1 /*
2 * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/shenandoahAsserts.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
32 #include "memory/iterator.inline.hpp"
33 #include "runtime/interfaceSupport.inline.hpp"
34 #ifdef COMPILER1
35 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
36 #endif
37 #ifdef COMPILER2
38 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
39 #endif
40
41 class ShenandoahBarrierSetC1;
42 class ShenandoahBarrierSetC2;
43
44 template <bool STOREVAL_EVAC_BARRIER>
45 class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure {
46 private:
47 ShenandoahHeap* _heap;
48 ShenandoahBarrierSet* _bs;
49
50 template <class T>
51 inline void do_oop_work(T* p) {
52 oop o;
53 if (STOREVAL_EVAC_BARRIER) {
54 o = _heap->evac_update_with_forwarded(p);
55 if (!CompressedOops::is_null(o)) {
56 _bs->enqueue(o);
57 }
58 } else {
59 _heap->maybe_update_with_forwarded(p);
60 }
61 }
62 public:
63 ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()), _bs(ShenandoahBarrierSet::barrier_set()) {
64 assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled");
65 }
66
67 virtual void do_oop(oop* p) { do_oop_work(p); }
68 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
69 };
70
71 ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) :
72 BarrierSet(make_barrier_set_assembler<ShenandoahBarrierSetAssembler>(),
73 make_barrier_set_c1<ShenandoahBarrierSetC1>(),
74 make_barrier_set_c2<ShenandoahBarrierSetC2>(),
75 NULL /* barrier_set_nmethod */,
76 BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)),
77 _heap(heap),
78 _satb_mark_queue_buffer_allocator("SATB Buffer Allocator", ShenandoahSATBBufferSize),
79 _satb_mark_queue_set(&_satb_mark_queue_buffer_allocator)
80 {
81 }
82
83 ShenandoahBarrierSetAssembler* ShenandoahBarrierSet::assembler() {
84 BarrierSetAssembler* const bsa = BarrierSet::barrier_set()->barrier_set_assembler();
85 return reinterpret_cast<ShenandoahBarrierSetAssembler*>(bsa);
86 }
87
88 void ShenandoahBarrierSet::print_on(outputStream* st) const {
89 st->print("ShenandoahBarrierSet");
90 }
91
92 bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) {
93 return bsn == BarrierSet::ShenandoahBarrierSet;
94 }
95
96 bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) {
97 return true;
98 }
99
100 template <class T, bool STOREVAL_EVAC_BARRIER>
101 void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) {
102 assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled");
103 ShenandoahUpdateRefsForOopClosure<STOREVAL_EVAC_BARRIER> cl;
104 T* dst = (T*) start;
105 for (size_t i = 0; i < count; i++) {
106 cl.do_oop(dst++);
107 }
108 }
109
110 void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) {
111 assert(_heap->is_update_refs_in_progress(), "should not be here otherwise");
112 assert(count > 0, "Should have been filtered before");
113
114 if (_heap->is_concurrent_traversal_in_progress()) {
115 ShenandoahEvacOOMScope oom_evac_scope;
116 if (UseCompressedOops) {
117 write_ref_array_loop<narrowOop, /* evac = */ true>(start, count);
118 } else {
119 write_ref_array_loop<oop, /* evac = */ true>(start, count);
120 }
121 } else {
122 if (UseCompressedOops) {
123 write_ref_array_loop<narrowOop, /* evac = */ false>(start, count);
124 } else {
125 write_ref_array_loop<oop, /* evac = */ false>(start, count);
126 }
127 }
128 }
129
130 template <class T>
131 void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, size_t count) {
132 shenandoah_assert_not_in_cset_loc_except(dst, _heap->cancelled_gc());
133 assert(ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).is_active(), "Shouldn't be here otherwise");
134 assert(ShenandoahSATBBarrier, "Shouldn't be here otherwise");
135 assert(count > 0, "Should have been filtered before");
136
137 Thread* thread = Thread::current();
138 ShenandoahMarkingContext* ctx = _heap->marking_context();
139 bool has_forwarded = _heap->has_forwarded_objects();
140 T* elem_ptr = dst;
141 for (size_t i = 0; i < count; i++, elem_ptr++) {
142 T heap_oop = RawAccess<>::oop_load(elem_ptr);
143 if (!CompressedOops::is_null(heap_oop)) {
144 oop obj = CompressedOops::decode_not_null(heap_oop);
145 if (has_forwarded) {
146 obj = resolve_forwarded_not_null(obj);
147 }
148 if (!ctx->is_marked(obj)) {
149 ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(obj);
150 }
151 }
152 }
153 }
154
155 void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized) {
156 if (! dest_uninitialized) {
157 write_ref_array_pre_work(dst, count);
158 }
159 }
160
161 void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, size_t count, bool dest_uninitialized) {
162 if (! dest_uninitialized) {
163 write_ref_array_pre_work(dst, count);
164 }
165 }
166
167 template <class T>
168 inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop new_val) {
169 shenandoah_assert_not_in_cset_loc_except(field, _heap->cancelled_gc());
170 if (_heap->is_concurrent_mark_in_progress()) {
171 T heap_oop = RawAccess<>::oop_load(field);
172 if (!CompressedOops::is_null(heap_oop)) {
173 enqueue(CompressedOops::decode(heap_oop));
174 }
175 }
176 }
177
178 // These are the more general virtual versions.
179 void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) {
180 inline_write_ref_field_pre(field, new_val);
181 }
182
183 void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) {
184 inline_write_ref_field_pre(field, new_val);
185 }
186
187 void ShenandoahBarrierSet::write_ref_field_pre_work(void* field, oop new_val) {
188 guarantee(false, "Not needed");
189 }
190
191 void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) {
192 shenandoah_assert_not_in_cset_loc_except(v, _heap->cancelled_gc());
193 shenandoah_assert_not_forwarded_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress());
194 shenandoah_assert_not_in_cset_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress());
195 }
196
197 void ShenandoahBarrierSet::write_region(MemRegion mr) {
198 if (!ShenandoahCloneBarrier) return;
199 if (!_heap->is_update_refs_in_progress()) return;
200
201 // This is called for cloning an object (see jvm.cpp) after the clone
202 // has been made. We are not interested in any 'previous value' because
203 // it would be NULL in any case. But we *are* interested in any oop*
204 // that potentially need to be updated.
205
206 oop obj = oop(mr.start());
207 shenandoah_assert_correct(NULL, obj);
208 if (_heap->is_concurrent_traversal_in_progress()) {
209 ShenandoahEvacOOMScope oom_evac_scope;
210 ShenandoahUpdateRefsForOopClosure</* evac = */ true> cl;
211 obj->oop_iterate(&cl);
212 } else {
213 ShenandoahUpdateRefsForOopClosure</* evac = */ false> cl;
214 obj->oop_iterate(&cl);
215 }
216 }
217
218 oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj) {
219 if (ShenandoahLoadRefBarrier && _heap->has_forwarded_objects()) {
220 return load_reference_barrier_impl(obj);
221 } else {
222 return obj;
223 }
224 }
225
226 oop ShenandoahBarrierSet::load_reference_barrier(oop obj) {
227 if (obj != NULL) {
228 return load_reference_barrier_not_null(obj);
229 } else {
230 return obj;
231 }
232 }
233
234
235 oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) {
236 assert(ShenandoahLoadRefBarrier, "should be enabled");
237 assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL), "evac should be in progress");
238 shenandoah_assert_in_cset(NULL, obj);
239
240 oop fwd = resolve_forwarded_not_null(obj);
241 if (obj == fwd) {
242 ShenandoahEvacOOMScope oom_evac_scope;
243
244 Thread* thread = Thread::current();
245 oop res_oop = _heap->evacuate_object(obj, thread);
246
247 // Since we are already here and paid the price of getting through runtime call adapters
248 // and acquiring oom-scope, it makes sense to try and evacuate more adjacent objects,
249 // thus amortizing the overhead. For sparsely live heaps, scan costs easily dominate
250 // total assist costs, and can introduce a lot of evacuation latency. This is why we
251 // only scan for _nearest_ N objects, regardless if they are eligible for evac or not.
252 // The scan itself should also avoid touching the non-marked objects below TAMS, because
253 // their metadata (notably, klasses) may be incorrect already.
254
255 size_t max = ShenandoahEvacAssist;
256 if (max > 0) {
257 // Traversal is special: it uses incomplete marking context, because it coalesces evac with mark.
258 // Other code uses complete marking context, because evac happens after the mark.
259 ShenandoahMarkingContext* ctx = _heap->is_concurrent_traversal_in_progress() ?
260 _heap->marking_context() : _heap->complete_marking_context();
261
262 ShenandoahHeapRegion* r = _heap->heap_region_containing(obj);
263 assert(r->is_cset(), "sanity");
264
265 HeapWord* cur = (HeapWord*)obj + obj->size();
266
267 size_t count = 0;
268 while ((cur < r->top()) && ctx->is_marked(oop(cur)) && (count++ < max)) {
269 oop cur_oop = oop(cur);
270 if (cur_oop == resolve_forwarded_not_null(cur_oop)) {
271 _heap->evacuate_object(cur_oop, thread);
272 }
273 cur = cur + cur_oop->size();
274 }
275 }
276
277 return res_oop;
278 }
279 return fwd;
280 }
281
282 oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj) {
283 assert(ShenandoahLoadRefBarrier, "should be enabled");
284 if (!CompressedOops::is_null(obj)) {
285 bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
286 oop fwd = resolve_forwarded_not_null(obj);
287 if (evac_in_progress &&
288 _heap->in_collection_set(obj) &&
289 obj == fwd) {
290 Thread *t = Thread::current();
291 if (t->is_GC_task_thread()) {
292 return _heap->evacuate_object(obj, t);
293 } else {
294 ShenandoahEvacOOMScope oom_evac_scope;
295 return _heap->evacuate_object(obj, t);
296 }
297 } else {
298 return fwd;
299 }
300 } else {
301 return obj;
302 }
303 }
304
305 void ShenandoahBarrierSet::storeval_barrier(oop obj) {
306 if (ShenandoahStoreValEnqueueBarrier && !CompressedOops::is_null(obj) && _heap->is_concurrent_traversal_in_progress()) {
307 enqueue(obj);
308 }
309 }
310
311 void ShenandoahBarrierSet::keep_alive_barrier(oop obj) {
312 if (ShenandoahKeepAliveBarrier && _heap->is_concurrent_mark_in_progress()) {
313 enqueue(obj);
314 }
315 }
316
317 void ShenandoahBarrierSet::enqueue(oop obj) {
318 shenandoah_assert_not_forwarded_if(NULL, obj, _heap->is_concurrent_traversal_in_progress());
319 assert(_satb_mark_queue_set.is_active(), "only get here when SATB active");
320
321 // Filter marked objects before hitting the SATB queues. The same predicate would
322 // be used by SATBMQ::filter to eliminate already marked objects downstream, but
323 // filtering here helps to avoid wasteful SATB queueing work to begin with.
324 if (!_heap->requires_marking<false>(obj)) return;
325
326 ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).enqueue_known_active(obj);
327 }
328
329 void ShenandoahBarrierSet::on_thread_create(Thread* thread) {
330 // Create thread local data
331 ShenandoahThreadLocalData::create(thread);
332 }
333
334 void ShenandoahBarrierSet::on_thread_destroy(Thread* thread) {
335 // Destroy thread local data
336 ShenandoahThreadLocalData::destroy(thread);
337 }
338
339 void ShenandoahBarrierSet::on_thread_attach(Thread *thread) {
340 assert(!thread->is_Java_thread() || !SafepointSynchronize::is_at_safepoint(),
341 "We should not be at a safepoint");
342 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
343 assert(!queue.is_active(), "SATB queue should not be active");
344 assert( queue.is_empty(), "SATB queue should be empty");
345 queue.set_active(_satb_mark_queue_set.is_active());
346 if (thread->is_Java_thread()) {
347 ShenandoahThreadLocalData::set_gc_state(thread, _heap->gc_state());
348 ShenandoahThreadLocalData::initialize_gclab(thread);
349 }
350 }
351
352 void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
353 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
354 queue.flush();
355 if (thread->is_Java_thread()) {
356 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
357 if (gclab != NULL) {
358 gclab->retire();
359 }
360 }
361 }
362
363 oop ShenandoahBarrierSet::oop_load_from_native_barrier(oop obj) {
364 if (CompressedOops::is_null(obj)) {
365 return NULL;
366 }
367
368 if (_heap->is_evacuation_in_progress() &&
369 !_heap->complete_marking_context()->is_marked(obj)) {
370 return NULL;
371 }
372
373 return load_reference_barrier_not_null(obj);
374 }