1 /*
2 * Copyright (c) 2017, 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24
25 #include "precompiled.hpp"
26 #include "code/codeCache.hpp"
27 #include "code/nmethod.hpp"
28 #include "gc_implementation/shenandoah/shenandoahHeap.hpp"
29 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
31
32 ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator() :
33 _claimed_idx(0), _finished(false) {
34 };
35
36 void ShenandoahParallelCodeCacheIterator::parallel_blobs_do(CodeBlobClosure* f) {
37 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
38
39 /*
40 * Parallel code heap walk.
41 *
42 * This code makes all threads scan all code heaps, but only one thread would execute the
43 * closure on given blob. This is achieved by recording the "claimed" blocks: if a thread
44 * had claimed the block, it can process all blobs in it. Others have to fast-forward to
45 * next attempt without processing.
46 *
47 * Late threads would return immediately if iterator is finished.
48 */
49
50 if (_finished) {
51 return;
52 }
53
54 int stride = 256; // educated guess
55 int stride_mask = stride - 1;
56 assert (is_power_of_2(stride), "sanity");
57
58 int count = 0;
59 bool process_block = true;
60
61 for (CodeBlob *cb = CodeCache::first(); cb != NULL; cb = CodeCache::next(cb)) {
62 int current = count++;
63 if ((current & stride_mask) == 0) {
64 process_block = (current >= _claimed_idx) &&
65 (Atomic::cmpxchg(current + stride, &_claimed_idx, current) == current);
66 }
67 if (process_block) {
68 if (cb->is_alive()) {
69 f->do_code_blob(cb);
70 #ifdef ASSERT
71 if (cb->is_nmethod())
72 ((nmethod*)cb)->verify_scavenge_root_oops();
73 #endif
74 }
75 }
76 }
77
78 _finished = true;
79 }
80
81 class ShenandoahNMethodOopDetector : public OopClosure {
82 private:
83 ShenandoahHeap* _heap;
84 GrowableArray<oop*> _oops;
85
86 public:
87 ShenandoahNMethodOopDetector() : _heap(ShenandoahHeap::heap()), _oops(10) {};
88
89 void do_oop(oop* o) {
90 _oops.append(o);
91 }
92
93 void do_oop(narrowOop* o) {
94 fatal("NMethods should not have compressed oops embedded.");
95 }
96
97 GrowableArray<oop*>* oops() {
98 return &_oops;
99 }
100
101 bool has_oops() {
102 return !_oops.is_empty();
103 }
104 };
105
106 class ShenandoahNMethodOopInitializer : public OopClosure {
107 public:
108 ShenandoahNMethodOopInitializer() {};
109
110 private:
111 template <class T>
112 inline void do_oop_work(T* p) {
113 T o = oopDesc::load_heap_oop(p);
114 if (! oopDesc::is_null(o)) {
115 oop obj1 = oopDesc::decode_heap_oop_not_null(o);
116 oop obj2 = oopDesc::bs()->write_barrier(obj1);
117 if (! oopDesc::unsafe_equals(obj1, obj2)) {
118 shenandoah_assert_not_in_cset(NULL, obj2);
119 oopDesc::encode_store_heap_oop(p, obj2);
120 }
121 }
122 }
123
124 public:
125 void do_oop(oop* o) {
126 do_oop_work(o);
127 }
128 void do_oop(narrowOop* o) {
129 do_oop_work(o);
130 }
131 };
132
133 volatile jint ShenandoahCodeRoots::_recorded_nms_lock;
134 GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;
135
136 void ShenandoahCodeRoots::initialize() {
137 _recorded_nms_lock = 0;
138 _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);
139 }
140
141 void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
142 switch (ShenandoahCodeRootsStyle) {
143 case 0:
144 case 1: {
145 ShenandoahNMethodOopInitializer init;
146 nm->oops_do(&init);
147 nm->fix_oop_relocations();
148 break;
149 }
150 case 2: {
151 ShenandoahNMethodOopDetector detector;
152 nm->oops_do(&detector);
153
154 if (detector.has_oops()) {
155 ShenandoahNMethodOopInitializer init;
156 nm->oops_do(&init);
157 nm->fix_oop_relocations();
158
159 ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
160 nmr->assert_alive_and_correct();
161
162 ShenandoahCodeRootsLock lock(true);
163
164 int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
165 if (idx != -1) {
166 ShenandoahNMethod* old = _recorded_nms->at(idx);
167 _recorded_nms->at_put(idx, nmr);
168 delete old;
169 } else {
170 _recorded_nms->append(nmr);
171 }
172 }
173 break;
174 }
175 default:
176 ShouldNotReachHere();
177 }
178 };
179
180 void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
181 switch (ShenandoahCodeRootsStyle) {
182 case 0:
183 case 1: {
184 break;
185 }
186 case 2: {
187 ShenandoahNMethodOopDetector detector;
188 nm->oops_do(&detector, /* allow_zombie = */ true);
189
190 if (detector.has_oops()) {
191 ShenandoahCodeRootsLock lock(true);
192
193 int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
194 assert(idx != -1, err_msg("nmethod " PTR_FORMAT " should be registered", p2i(nm)));
195 ShenandoahNMethod* old = _recorded_nms->at(idx);
196 old->assert_same_oops(detector.oops());
197 _recorded_nms->delete_at(idx);
198 delete old;
199 }
200 break;
201 }
202 default:
203 ShouldNotReachHere();
204 }
205 }
206
207 ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
208 _heap(ShenandoahHeap::heap()),
209 _claimed(0) {
210 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
211 switch (ShenandoahCodeRootsStyle) {
212 case 0:
213 case 1:
214 break;
215 case 2:
216 ShenandoahCodeRoots::acquire_lock(false);
217 break;
218 default:
219 ShouldNotReachHere();
220 }
221 }
222
223 ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
224 switch (ShenandoahCodeRootsStyle) {
225 case 0:
226 case 1: {
227 // No need to do anything here
228 break;
229 }
230 case 2: {
231 ShenandoahCodeRoots::release_lock(false);
232 break;
233 }
234 default:
235 ShouldNotReachHere();
236 }
237 }
238
239 template<bool CSET_FILTER>
240 void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
241 switch (ShenandoahCodeRootsStyle) {
242 case 0: {
243 if (_seq_claimed.try_set()) {
244 CodeCache::blobs_do(f);
245 }
246 break;
247 }
248 case 1: {
249 _par_iterator.parallel_blobs_do(f);
250 break;
251 }
252 case 2: {
253 ShenandoahCodeRootsIterator::fast_parallel_blobs_do<CSET_FILTER>(f);
254 break;
255 }
256 default:
257 ShouldNotReachHere();
258 }
259 }
260
261 ShenandoahAllCodeRootsIterator ShenandoahCodeRoots::iterator() {
262 return ShenandoahAllCodeRootsIterator();
263 }
264
265 ShenandoahCsetCodeRootsIterator ShenandoahCodeRoots::cset_iterator() {
266 return ShenandoahCsetCodeRootsIterator();
267 }
268
269 void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
270 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
271 }
272
273 void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
274 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
275 }
276
277 template <bool CSET_FILTER>
278 void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
279 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
280
281 size_t stride = 256; // educated guess
282
283 GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;
284
285 jlong max = list->length();
286 while (_claimed < max) {
287 size_t cur = (size_t)Atomic::add(stride, &_claimed); // Note: in JDK 8, add() returns old value
288 size_t start = cur;
289 size_t end = MIN2(cur + stride, (size_t) max);
290 if (start >= (size_t)max) break;
291
292 for (size_t idx = start; idx < end; idx++) {
293 ShenandoahNMethod* nmr = list->at((int) idx);
294 nmr->assert_alive_and_correct();
295
296 if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {
297 continue;
298 }
299
300 f->do_code_blob(nmr->nm());
301 }
302 }
303 }
304
305 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {
306 _nm = nm;
307 _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);
308 _oops_count = oops->length();
309 for (int c = 0; c < _oops_count; c++) {
310 _oops[c] = oops->at(c);
311 }
312 }
313
314 ShenandoahNMethod::~ShenandoahNMethod() {
315 if (_oops != NULL) {
316 FREE_C_HEAP_ARRAY(oop*, _oops, mtGC);
317 }
318 }
319
320 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
321 for (int c = 0; c < _oops_count; c++) {
322 oop o = oopDesc::load_heap_oop(_oops[c]);
323 if (heap->in_collection_set(o)) {
324 return true;
325 }
326 }
327 return false;
328 }
329
330 #ifdef ASSERT
331 void ShenandoahNMethod::assert_alive_and_correct() {
332 assert(_nm->is_alive(), "only alive nmethods here");
333 assert(_oops_count > 0, "should have filtered nmethods without oops before");
334 ShenandoahHeap* heap = ShenandoahHeap::heap();
335 for (int c = 0; c < _oops_count; c++) {
336 oop o = oopDesc::load_heap_oop(_oops[c]);
337 shenandoah_assert_correct_except(NULL, o, o == NULL || heap->is_full_gc_move_in_progress());
338 assert(_nm->code_contains((address)_oops[c]) || _nm->oops_contains(_oops[c]), "nmethod should contain the oop*");
339 }
340 }
341
342 void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {
343 assert(_oops_count == oops->length(), "should have the same number of oop*");
344 for (int c = 0; c < _oops_count; c++) {
345 assert(_oops[c] == oops->at(c), "should be the same oop*");
346 }
347 }
348 #endif