1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
28 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
29 #include "runtime/interfaceSupport.hpp"
30
31 class UpdateRefsForOopClosure: public ExtendedOopClosure {
32
33 private:
34 ShenandoahHeap* _heap;
35 template <class T>
36 inline void do_oop_work(T* p) {
37 _heap->maybe_update_oop_ref(p);
38 }
39 public:
40 UpdateRefsForOopClosure() {
41 _heap = ShenandoahHeap::heap();
42 }
43
44 void do_oop(oop* p) {
45 do_oop_work(p);
46 }
47
48 void do_oop(narrowOop* p) {
49 do_oop_work(p);
50 }
51
52 };
53
54 class UpdateRefsForOopMatrixClosure: public ExtendedOopClosure {
55
56 private:
57 ShenandoahHeap* _heap;
58 template <class T>
59 inline void do_oop_work(T* p) {
60 oop o = _heap->maybe_update_oop_ref(p);
61 if (! oopDesc::is_null(o)) {
62 _heap->connection_matrix()->set_connected(_heap->heap_region_index_containing(p), _heap->heap_region_index_containing(o), true);
63 }
64 }
65 public:
66 UpdateRefsForOopMatrixClosure() {
67 _heap = ShenandoahHeap::heap();
68 }
69
70 void do_oop(oop* p) {
71 do_oop_work(p);
72 }
73
74 void do_oop(narrowOop* p) {
75 do_oop_work(p);
76 }
77
78 };
79
80 ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) :
81 BarrierSet(BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)),
82 _heap(heap)
83 {
84 }
85
86 void ShenandoahBarrierSet::print_on(outputStream* st) const {
87 st->print("ShenandoahBarrierSet");
88 }
89
90 bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) {
91 return bsn == BarrierSet::ShenandoahBarrierSet;
92 }
93
94 bool ShenandoahBarrierSet::has_read_prim_array_opt() {
95 return true;
96 }
97
98 bool ShenandoahBarrierSet::has_read_prim_barrier() {
99 return false;
100 }
101
102 bool ShenandoahBarrierSet::has_read_ref_array_opt() {
103 return true;
104 }
105
106 bool ShenandoahBarrierSet::has_read_ref_barrier() {
107 return false;
108 }
109
110 bool ShenandoahBarrierSet::has_read_region_opt() {
111 return true;
112 }
113
114 bool ShenandoahBarrierSet::has_write_prim_array_opt() {
115 return true;
116 }
117
118 bool ShenandoahBarrierSet::has_write_prim_barrier() {
119 return false;
120 }
121
122 bool ShenandoahBarrierSet::has_write_ref_array_opt() {
123 return true;
124 }
125
126 bool ShenandoahBarrierSet::has_write_ref_barrier() {
127 return true;
128 }
129
130 bool ShenandoahBarrierSet::has_write_ref_pre_barrier() {
131 return true;
132 }
133
134 bool ShenandoahBarrierSet::has_write_region_opt() {
135 return true;
136 }
137
138 bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) {
139 return true;
140 }
141
142 void ShenandoahBarrierSet::read_prim_array(MemRegion mr) {
143 Unimplemented();
144 }
145
146 void ShenandoahBarrierSet::read_prim_field(HeapWord* hw, size_t s){
147 Unimplemented();
148 }
149
150 bool ShenandoahBarrierSet::read_prim_needs_barrier(HeapWord* hw, size_t s) {
151 return false;
152 }
153
154 void ShenandoahBarrierSet::read_ref_array(MemRegion mr) {
155 Unimplemented();
156 }
157
158 void ShenandoahBarrierSet::read_ref_field(void* v) {
159 // tty->print_cr("read_ref_field: v = "PTR_FORMAT, v);
160 // return *v;
161 }
162
163 bool ShenandoahBarrierSet::read_ref_needs_barrier(void* v) {
164 Unimplemented();
165 return false;
166 }
167
168 void ShenandoahBarrierSet::read_region(MemRegion mr) {
169 Unimplemented();
170 }
171
172 void ShenandoahBarrierSet::resize_covered_region(MemRegion mr) {
173 Unimplemented();
174 }
175
176 void ShenandoahBarrierSet::write_prim_array(MemRegion mr) {
177 Unimplemented();
178 }
179
180 void ShenandoahBarrierSet::write_prim_field(HeapWord* hw, size_t s , juint x, juint y) {
181 Unimplemented();
182 }
183
184 bool ShenandoahBarrierSet::write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y) {
185 Unimplemented();
186 return false;
187 }
188
189 bool ShenandoahBarrierSet::need_update_refs_barrier() {
190 return UseShenandoahMatrix || (_heap->concurrent_mark_in_progress() && _heap->need_update_refs());
191 }
192
193 void ShenandoahBarrierSet::write_ref_array_work(MemRegion r) {
194 ShouldNotReachHere();
195 }
196
197 void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) {
198 if (! need_update_refs_barrier()) return;
199 ShenandoahHeap* heap = ShenandoahHeap::heap();
200 ShenandoahConnectionMatrix* matrix = heap->connection_matrix();
201 // TODO: Use templated loop and split on oop/narrowOop and UseShenandoahMatrix.
202 if (UseCompressedOops) {
203 narrowOop* dst = (narrowOop*) start;
204 for (size_t i = 0; i < count; i++, dst++) {
205 oop o = _heap->maybe_update_oop_ref(dst);
206 if (UseShenandoahMatrix && ! oopDesc::is_null(o)) {
207 matrix->set_connected(heap->heap_region_index_containing(dst), heap->heap_region_index_containing(o), true);
208 }
209 }
210 } else {
211 oop* dst = (oop*) start;
212 for (size_t i = 0; i < count; i++, dst++) {
213 oop o = _heap->maybe_update_oop_ref(dst);
214 if (UseShenandoahMatrix && ! oopDesc::is_null(o)) {
215 matrix->set_connected(heap->heap_region_index_containing(dst), heap->heap_region_index_containing(o), true);
216 }
217 }
218 }
219 }
220
221 template <class T>
222 void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, int count) {
223
224 #ifdef ASSERT
225 if (_heap->is_in(dst) &&
226 _heap->in_collection_set(dst) &&
227 ! _heap->cancelled_concgc()) {
228 tty->print_cr("dst = "PTR_FORMAT, p2i(dst));
229 _heap->heap_region_containing((HeapWord*) dst)->print();
230 assert(false, "We should have fixed this earlier");
231 }
232 #endif
233
234 if (! JavaThread::satb_mark_queue_set().is_active()) return;
235 T* elem_ptr = dst;
236 for (int i = 0; i < count; i++, elem_ptr++) {
237 T heap_oop = oopDesc::load_heap_oop(elem_ptr);
238 if (!oopDesc::is_null(heap_oop)) {
239 G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop_not_null(heap_oop));
240 }
241 }
242 }
243
244 void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) {
245 if (! dest_uninitialized) {
246 write_ref_array_pre_work(dst, count);
247 }
248 }
249
250 void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) {
251 if (! dest_uninitialized) {
252 write_ref_array_pre_work(dst, count);
253 }
254 }
255
256 template <class T>
257 void ShenandoahBarrierSet::write_ref_field_pre_static(T* field, oop new_val) {
258 T heap_oop = oopDesc::load_heap_oop(field);
259
260 #ifdef ASSERT
261 {
262 ShenandoahHeap* heap = ShenandoahHeap::heap();
263 if (heap->is_in(field) &&
264 heap->in_collection_set(field) &&
265 ! heap->cancelled_concgc()) {
266 tty->print_cr("field = "PTR_FORMAT, p2i(field));
267 tty->print_cr("in_cset: %s", BOOL_TO_STR(heap->in_collection_set(field)));
268 heap->heap_region_containing((HeapWord*)field)->print();
269 tty->print_cr("marking: %s, evacuating: %s",
270 BOOL_TO_STR(heap->concurrent_mark_in_progress()),
271 BOOL_TO_STR(heap->is_evacuation_in_progress()));
272 assert(false, "We should have fixed this earlier");
273 }
274 }
275 #endif
276
277 if (!oopDesc::is_null(heap_oop)) {
278 G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop(heap_oop));
279 }
280 if (UseShenandoahMatrix && ! oopDesc::is_null(new_val)) {
281 ShenandoahHeap* heap = ShenandoahHeap::heap();
282 ShenandoahConnectionMatrix* matrix = heap->connection_matrix();
283 matrix->set_connected(heap->heap_region_index_containing(field), heap->heap_region_index_containing(new_val), true);
284 }
285 }
286
287 template <class T>
288 inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) {
289 write_ref_field_pre_static(field, newVal);
290 }
291
292 // These are the more general virtual versions.
293 void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) {
294 write_ref_field_pre_static(field, new_val);
295 }
296
297 void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) {
298 write_ref_field_pre_static(field, new_val);
299 }
300
301 void ShenandoahBarrierSet::write_ref_field_pre_work(void* field, oop new_val) {
302 guarantee(false, "Not needed");
303 }
304
305 void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) {
306 #ifdef ASSERT
307 ShenandoahHeap* heap = ShenandoahHeap::heap();
308 if (!(heap->cancelled_concgc() || !heap->in_collection_set(v))) {
309 tty->print_cr("field not in collection set: "PTR_FORMAT, p2i(v));
310 tty->print_cr("containing heap region:");
311 ShenandoahHeap::heap()->heap_region_containing(v)->print();
312 }
313 assert(heap->cancelled_concgc() || !heap->in_collection_set(v), "only write to to-space");
314 if (_heap->concurrent_mark_in_progress()) {
315 assert(o == NULL || oopDesc::unsafe_equals(o, resolve_oop_static(o)), "only write to-space values");
316 assert(o == NULL || !heap->in_collection_set(o), "only write to-space values");
317 }
318 #endif
319 if (UseShenandoahMatrix && ! oopDesc::is_null(o)) {
320 ShenandoahHeap* heap = ShenandoahHeap::heap();
321 ShenandoahConnectionMatrix* matrix = heap->connection_matrix();
322 matrix->set_connected(heap->heap_region_index_containing(v), heap->heap_region_index_containing(o), true);
323 }
324 }
325
326 void ShenandoahBarrierSet::write_region_work(MemRegion mr) {
327
328 if (! need_update_refs_barrier()) return;
329
330 // This is called for cloning an object (see jvm.cpp) after the clone
331 // has been made. We are not interested in any 'previous value' because
332 // it would be NULL in any case. But we *are* interested in any oop*
333 // that potentially need to be updated.
334
335 oop obj = oop(mr.start());
336 assert(obj->is_oop(), "must be an oop");
337 if (UseShenandoahMatrix) {
338 UpdateRefsForOopMatrixClosure cl;
339 obj->oop_iterate(&cl);
340 } else {
341 UpdateRefsForOopClosure cl;
342 obj->oop_iterate(&cl);
343 }
344 }
345
346 oop ShenandoahBarrierSet::read_barrier(oop src) {
347 if (ShenandoahReadBarrier) {
348 return ShenandoahBarrierSet::resolve_oop_static(src);
349 } else {
350 return src;
351 }
352 }
353
354 bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) {
355 bool eq = oopDesc::unsafe_equals(obj1, obj2);
356 if (! eq && ShenandoahAcmpBarrier) {
357 OrderAccess::loadload();
358 obj1 = resolve_oop_static(obj1);
359 obj2 = resolve_oop_static(obj2);
360 eq = oopDesc::unsafe_equals(obj1, obj2);
361 }
362 return eq;
363 }
364
365 bool ShenandoahBarrierSet::obj_equals(narrowOop obj1, narrowOop obj2) {
366 return obj_equals(oopDesc::decode_heap_oop(obj1), oopDesc::decode_heap_oop(obj2));
367 }
368
369 #ifdef ASSERT
370 bool ShenandoahBarrierSet::is_safe(oop o) {
371 if (o == NULL) return true;
372 if (_heap->in_collection_set(o)) {
373 return false;
374 }
375 if (! oopDesc::unsafe_equals(o, read_barrier(o))) {
376 return false;
377 }
378 return true;
379 }
380
381 bool ShenandoahBarrierSet::is_safe(narrowOop o) {
382 return is_safe(oopDesc::decode_heap_oop(o));
383 }
384 #endif
385
386 oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop_work(oop src) {
387 assert(src != NULL, "only evacuated non NULL oops");
388
389 if (_heap->in_collection_set(src)) {
390 return resolve_and_maybe_copy_oop_work2(src);
391 } else {
392 return src;
393 }
394 }
395
396 oop ShenandoahBarrierSet::resolve_and_maybe_copy_oop_work2(oop src) {
397 assert(src != NULL, "only evacuated non NULL oops");
398 assert(_heap->in_collection_set(src), "only evacuate objects in collection set");
399 assert(! _heap->heap_region_containing(src)->is_humongous(), "never evacuate humongous objects");
400 // TODO: Consider passing thread from caller.
401 bool evac;
402 oop dst = _heap->evacuate_object(src, Thread::current(), evac);
403
404 log_develop_trace(gc, compaction)("src = "PTR_FORMAT" dst = "PTR_FORMAT" src-2 = "PTR_FORMAT,
405 p2i(src), p2i(dst), p2i(((HeapWord*) src) - 2));
406
407 assert(_heap->is_in(dst), "result should be in the heap");
408 return dst;
409 }
410
411 oop ShenandoahBarrierSet::resolve_and_maybe_copy_oopHelper(oop src) {
412 assert(src != NULL, "checked before");
413 bool evac = _heap->is_evacuation_in_progress();
414 OrderAccess::loadload();
415 src = resolve_oop_static_not_null(src);
416 if (! evac) {
417 return src;
418 } else {
419 return resolve_and_maybe_copy_oop_work(src);
420 }
421 }
422
423 JRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_c2(oopDesc* src))
424 oop result = ((ShenandoahBarrierSet*) oopDesc::bs())->resolve_and_maybe_copy_oop_work2(src);
425 return (oopDesc*) result;
426 JRT_END
427
428 oop ShenandoahBarrierSet::write_barrier(oop src) {
429 if (ShenandoahWriteBarrier) {
430 if (!oopDesc::is_null(src)) {
431 assert(_heap->is_in(src), "sanity");
432 assert(src != NULL, "checked before");
433 oop result = resolve_and_maybe_copy_oopHelper(src);
434 assert(_heap->is_in(result) /*&& result->is_oop()*/, "resolved oop must be NULL, or a valid oop in the heap");
435 return result;
436 } else {
437 return NULL;
438 }
439 } else {
440 return src;
441 }
442 }
443
444 #ifdef ASSERT
445 void ShenandoahBarrierSet::verify_safe_oop(oop p) {
446 ShenandoahHeap* heap = ShenandoahHeap::heap();
447 if (p == NULL) return;
448 if (heap->in_collection_set(p) &&
449 ! heap->cancelled_concgc()) {
450 tty->print_cr("oop = "PTR_FORMAT", resolved: "PTR_FORMAT", marked-next %s, marked-complete: %s",
451 p2i(p),
452 p2i(read_barrier(p)),
453 BOOL_TO_STR(heap->is_marked_next(p)),
454 BOOL_TO_STR(heap->is_marked_complete(p)));
455 tty->print_cr("in_cset: %s", BOOL_TO_STR(heap->in_collection_set(p)));
456 heap->heap_region_containing((HeapWord*) p)->print();
457 tty->print_cr("top-at-mark-start: %p", heap->next_top_at_mark_start((HeapWord*) p));
458 tty->print_cr("top-at-prev-mark-start: %p", heap->complete_top_at_mark_start((HeapWord*) p));
459 tty->print_cr("marking: %s, evacuating: %s", BOOL_TO_STR(heap->concurrent_mark_in_progress()), BOOL_TO_STR(heap->is_evacuation_in_progress()));
460 assert(false, "We should have fixed this earlier");
461 }
462 }
463 #endif