1 /*
2 * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "code/codeCache.hpp"
26 #include "code/nmethod.hpp"
27 #include "gc_implementation/shenandoah/shenandoahHeap.hpp"
28 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
29 #include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
30 #include "memory/resourceArea.hpp"
31
32 ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator() :
33 _claimed_idx(0), _finished(false) {
34 }
35
36 void ShenandoahParallelCodeCacheIterator::parallel_blobs_do(CodeBlobClosure* f) {
37 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
38
39 /*
40 * Parallel code heap walk.
41 *
42 * This code makes all threads scan all code heaps, but only one thread would execute the
43 * closure on given blob. This is achieved by recording the "claimed" blocks: if a thread
44 * had claimed the block, it can process all blobs in it. Others have to fast-forward to
45 * next attempt without processing.
46 *
47 * Late threads would return immediately if iterator is finished.
48 */
49
50 if (_finished) {
51 return;
52 }
53
54 int stride = 256; // educated guess
55 int stride_mask = stride - 1;
56 assert (is_power_of_2(stride), "sanity");
57
58 int count = 0;
59 bool process_block = true;
60
61 for (CodeBlob *cb = CodeCache::first(); cb != NULL; cb = CodeCache::next(cb)) {
62 int current = count++;
63 if ((current & stride_mask) == 0) {
64 process_block = (current >= _claimed_idx) &&
65 (Atomic::cmpxchg(current + stride, &_claimed_idx, current) == current);
66 }
67 if (process_block) {
68 if (cb->is_alive()) {
69 f->do_code_blob(cb);
70 #ifdef ASSERT
71 if (cb->is_nmethod())
72 ((nmethod*)cb)->verify_scavenge_root_oops();
73 #endif
74 }
75 }
76 }
77
78 _finished = true;
79 }
80
81 class ShenandoahNMethodOopDetector : public OopClosure {
82 private:
83 ResourceMark rm; // For growable array allocation below.
84 GrowableArray<oop*> _oops;
85
86 public:
87 ShenandoahNMethodOopDetector() : _oops(10) {};
88
89 void do_oop(oop* o) {
90 _oops.append(o);
91 }
92
93 void do_oop(narrowOop* o) {
94 fatal("NMethods should not have compressed oops embedded.");
95 }
96
97 GrowableArray<oop*>* oops() {
98 return &_oops;
99 }
100
101 bool has_oops() {
102 return !_oops.is_empty();
103 }
104 };
105
106 class ShenandoahNMethodOopInitializer : public OopClosure {
107 public:
108 ShenandoahNMethodOopInitializer() {};
109
110 private:
111 template <class T>
112 inline void do_oop_work(T* p) {
113 T o = oopDesc::load_heap_oop(p);
114 if (! oopDesc::is_null(o)) {
115 oop obj1 = oopDesc::decode_heap_oop_not_null(o);
116 oop obj2 = oopDesc::bs()->write_barrier(obj1);
117 if (! oopDesc::unsafe_equals(obj1, obj2)) {
118 shenandoah_assert_not_in_cset(NULL, obj2);
119 oopDesc::encode_store_heap_oop(p, obj2);
120 }
121 }
122 }
123
124 public:
125 void do_oop(oop* o) {
126 do_oop_work(o);
127 }
128 void do_oop(narrowOop* o) {
129 do_oop_work(o);
130 }
131 };
132
133 ShenandoahCodeRoots::PaddedLock ShenandoahCodeRoots::_recorded_nms_lock;
134 GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;
135
136 void ShenandoahCodeRoots::initialize() {
137 _recorded_nms_lock._lock = 0;
138 _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);
139 }
140
141 void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
142 switch (ShenandoahCodeRootsStyle) {
143 case 0:
144 case 1: {
145 ShenandoahNMethodOopInitializer init;
146 nm->oops_do(&init);
147 nm->fix_oop_relocations();
148 break;
149 }
150 case 2: {
151 ShenandoahNMethodOopDetector detector;
152 nm->oops_do(&detector);
153
154 if (detector.has_oops()) {
155 ShenandoahNMethodOopInitializer init;
156 nm->oops_do(&init);
157 nm->fix_oop_relocations();
158
159 ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
160 nmr->assert_alive_and_correct();
161
162 ShenandoahCodeRootsLock lock(true);
163
164 int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
165 if (idx != -1) {
166 ShenandoahNMethod* old = _recorded_nms->at(idx);
167 _recorded_nms->at_put(idx, nmr);
168 delete old;
169 } else {
170 _recorded_nms->append(nmr);
171 }
172 }
173 break;
174 }
175 default:
176 ShouldNotReachHere();
177 }
178 };
179
180 void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
181 switch (ShenandoahCodeRootsStyle) {
182 case 0:
183 case 1: {
184 break;
185 }
186 case 2: {
187 ShenandoahNMethodOopDetector detector;
188 nm->oops_do(&detector, /* allow_zombie = */ true);
189
190 if (detector.has_oops()) {
191 ShenandoahCodeRootsLock lock(true);
192
193 int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
194 assert(idx != -1, err_msg("nmethod " PTR_FORMAT " should be registered", p2i(nm)));
195 ShenandoahNMethod* old = _recorded_nms->at(idx);
196 old->assert_same_oops(detector.oops());
197 _recorded_nms->delete_at(idx);
198 delete old;
199 }
200 break;
201 }
202 default:
203 ShouldNotReachHere();
204 }
205 }
206
207 ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
208 _heap(ShenandoahHeap::heap()),
209 _claimed(0) {
210 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
211 assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers");
212 switch (ShenandoahCodeRootsStyle) {
213 case 0:
214 case 1:
215 break;
216 case 2:
217 ShenandoahCodeRoots::acquire_lock(false);
218 break;
219 default:
220 ShouldNotReachHere();
221 }
222 }
223
224 ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
225 switch (ShenandoahCodeRootsStyle) {
226 case 0:
227 case 1: {
228 // No need to do anything here
229 break;
230 }
231 case 2: {
232 ShenandoahCodeRoots::release_lock(false);
233 break;
234 }
235 default:
236 ShouldNotReachHere();
237 }
238 }
239
240 template<bool CSET_FILTER>
241 void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
242 switch (ShenandoahCodeRootsStyle) {
243 case 0: {
244 if (_seq_claimed.try_set()) {
245 CodeCache::blobs_do(f);
246 }
247 break;
248 }
249 case 1: {
250 _par_iterator.parallel_blobs_do(f);
251 break;
252 }
253 case 2: {
254 ShenandoahCodeRootsIterator::fast_parallel_blobs_do<CSET_FILTER>(f);
255 break;
256 }
257 default:
258 ShouldNotReachHere();
259 }
260 }
261
262 ShenandoahAllCodeRootsIterator ShenandoahCodeRoots::iterator() {
263 return ShenandoahAllCodeRootsIterator();
264 }
265
266 ShenandoahCsetCodeRootsIterator ShenandoahCodeRoots::cset_iterator() {
267 return ShenandoahCsetCodeRootsIterator();
268 }
269
270 void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
271 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
272 }
273
274 void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
275 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
276 }
277
278 template <bool CSET_FILTER>
279 void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
280 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
281
282 size_t stride = 256; // educated guess
283
284 GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;
285
286 jlong max = list->length();
287 while (_claimed < max) {
288 size_t cur = (size_t)Atomic::add(stride, &_claimed); // Note: in JDK 8, add() returns old value
289 size_t start = cur;
290 size_t end = MIN2(cur + stride, (size_t) max);
291 if (start >= (size_t)max) break;
292
293 for (size_t idx = start; idx < end; idx++) {
294 ShenandoahNMethod* nmr = list->at((int) idx);
295 nmr->assert_alive_and_correct();
296
297 if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {
298 continue;
299 }
300
301 f->do_code_blob(nmr->nm());
302 }
303 }
304 }
305
306 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {
307 _nm = nm;
308 _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);
309 _oops_count = oops->length();
310 for (int c = 0; c < _oops_count; c++) {
311 _oops[c] = oops->at(c);
312 }
313 }
314
315 ShenandoahNMethod::~ShenandoahNMethod() {
316 if (_oops != NULL) {
317 FREE_C_HEAP_ARRAY(oop*, _oops, mtGC);
318 }
319 }
320
321 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
322 for (int c = 0; c < _oops_count; c++) {
323 oop o = oopDesc::load_heap_oop(_oops[c]);
324 if (heap->in_collection_set(o)) {
325 return true;
326 }
327 }
328 return false;
329 }
330
331 #ifdef ASSERT
332 void ShenandoahNMethod::assert_alive_and_correct() {
333 assert(_nm->is_alive(), "only alive nmethods here");
334 assert(_oops_count > 0, "should have filtered nmethods without oops before");
335 ShenandoahHeap* heap = ShenandoahHeap::heap();
336 for (int c = 0; c < _oops_count; c++) {
337 oop *loc = _oops[c];
338 assert(_nm->code_contains((address)loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
339 oop o = oopDesc::load_heap_oop(loc);
340 shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress());
341 }
342 }
343
344 void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {
345 assert(_oops_count == oops->length(), "should have the same number of oop*");
346 for (int c = 0; c < _oops_count; c++) {
347 assert(_oops[c] == oops->at(c), "should be the same oop*");
348 }
349 }
350 #endif