1 /*
2 * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP
25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP
26
27 #include "gc_implementation/shenandoah/brooksPointer.hpp"
28 #include "gc_implementation/shenandoah/shenandoahAsserts.hpp"
29 #include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp"
30 #include "gc_implementation/shenandoah/shenandoahConcurrentMark.hpp"
31 #include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp"
32 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc_implementation/shenandoah/shenandoahStringDedup.hpp"
34 #include "gc_implementation/shenandoah/shenandoahTaskqueue.inline.hpp"
35 #include "memory/iterator.inline.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "runtime/prefetch.inline.hpp"
38
39 template <class T>
40 void ShenandoahConcurrentMark::do_task(ShenandoahObjToScanQueue* q, T* cl, jushort* live_data, ShenandoahMarkTask* task) {
41 oop obj = task->obj();
42
43 shenandoah_assert_not_forwarded(NULL, obj);
44 shenandoah_assert_marked_next(NULL, obj);
45 shenandoah_assert_not_in_cset_except(NULL, obj, _heap->cancelled_gc());
46
47 if (task->is_not_chunked()) {
48 count_liveness(live_data, obj);
49 if (obj->is_instance()) {
50 // Case 1: Normal oop, process as usual.
51 obj->oop_iterate(cl);
52 } else if (obj->is_objArray()) {
53 // Case 2: Object array instance and no chunk is set. Must be the first
54 // time we visit it, start the chunked processing.
55 do_chunked_array_start<T>(q, cl, obj);
56 } else {
57 // Case 3: Primitive array. Do nothing, no oops there. We use the same
58 // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using:
59 // We skip iterating over the klass pointer since we know that
60 // Universe::TypeArrayKlass never moves.
61 assert (obj->is_typeArray(), "should be type array");
62 }
63 } else {
64 // Case 4: Array chunk, has sensible chunk id. Process it.
65 do_chunked_array<T>(q, cl, obj, task->chunk(), task->pow());
66 }
67 }
68
69 inline void ShenandoahConcurrentMark::count_liveness(jushort* live_data, oop obj) {
70 size_t region_idx = _heap->heap_region_index_containing(obj);
71 ShenandoahHeapRegion* region = _heap->get_region(region_idx);
72 if (!region->is_humongous_start()) {
73 assert(!region->is_humongous(), "Cannot have continuations here");
74 jushort cur = live_data[region_idx];
75 size_t size = obj->size() + BrooksPointer::word_size();
76 size_t max = (1 << (sizeof(jushort) * 8)) - 1;
77 if (size >= max) {
78 // too big, add to region data directly
79 region->increase_live_data_gc_words(size);
80 } else {
81 size_t new_val = cur + size;
82 if (new_val >= max) {
83 // overflow, flush to region data
84 region->increase_live_data_gc_words(new_val);
85 live_data[region_idx] = 0;
86 } else {
87 // still good, remember in locals
88 live_data[region_idx] = (jushort) new_val;
89 }
90 }
91 } else {
92 count_liveness_humongous(obj);
93 }
94 }
95
96 inline void ShenandoahConcurrentMark::count_liveness_humongous(oop obj) {
97 shenandoah_assert_in_correct_region(NULL, obj);
98 size_t region_idx = _heap->heap_region_index_containing(obj);
99 size_t size = obj->size() + BrooksPointer::word_size();
100 size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
101
102 for (size_t i = region_idx; i < region_idx + num_regions; i++) {
103 ShenandoahHeapRegion* chain_reg = _heap->get_region(i);
104 assert(chain_reg->is_humongous(), "Expecting a humongous region");
105 chain_reg->increase_live_data_gc_words(chain_reg->used() >> LogHeapWordSize);
106 }
107 }
108
109 template <class T>
110 inline void ShenandoahConcurrentMark::do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop obj) {
111 assert(obj->is_objArray(), "expect object array");
112 objArrayOop array = objArrayOop(obj);
113 int len = array->length();
114
115 if (len <= (int) ObjArrayMarkingStride*2) {
116 // A few slices only, process directly
117 array->oop_iterate_range(cl, 0, len);
118 } else {
119 int bits = log2_long(len);
120 // Compensate for non-power-of-two arrays, cover the array in excess:
121 if (len != (1 << bits)) bits++;
122
123 // Only allow full chunks on the queue. This frees do_chunked_array() from checking from/to
124 // boundaries against array->length(), touching the array header on every chunk.
125 //
126 // To do this, we cut the prefix in full-sized chunks, and submit them on the queue.
127 // If the array is not divided in chunk sizes, then there would be an irregular tail,
128 // which we will process separately.
129
130 int last_idx = 0;
131
132 int chunk = 1;
133 int pow = bits;
134
135 // Handle overflow
136 if (pow >= 31) {
137 assert (pow == 31, "sanity");
138 pow--;
139 chunk = 2;
140 last_idx = (1 << pow);
141 bool pushed = q->push(ShenandoahMarkTask(array, 1, pow));
142 assert(pushed, "overflow queue should always succeed pushing");
143 }
144
145 // Split out tasks, as suggested in ObjArrayChunkedTask docs. Record the last
146 // successful right boundary to figure out the irregular tail.
147 while ((1 << pow) > (int)ObjArrayMarkingStride &&
148 (chunk*2 < ShenandoahMarkTask::chunk_size())) {
149 pow--;
150 int left_chunk = chunk*2 - 1;
151 int right_chunk = chunk*2;
152 int left_chunk_end = left_chunk * (1 << pow);
153 if (left_chunk_end < len) {
154 bool pushed = q->push(ShenandoahMarkTask(array, left_chunk, pow));
155 assert(pushed, "overflow queue should always succeed pushing");
156 chunk = right_chunk;
157 last_idx = left_chunk_end;
158 } else {
159 chunk = left_chunk;
160 }
161 }
162
163 // Process the irregular tail, if present
164 int from = last_idx;
165 if (from < len) {
166 array->oop_iterate_range(cl, from, len);
167 }
168 }
169 }
170
171 template <class T>
172 inline void ShenandoahConcurrentMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop obj, int chunk, int pow) {
173 assert(obj->is_objArray(), "expect object array");
174 objArrayOop array = objArrayOop(obj);
175
176 assert (ObjArrayMarkingStride > 0, "sanity");
177
178 // Split out tasks, as suggested in ObjArrayChunkedTask docs. Avoid pushing tasks that
179 // are known to start beyond the array.
180 while ((1 << pow) > (int)ObjArrayMarkingStride && (chunk*2 < ShenandoahMarkTask::chunk_size())) {
181 pow--;
182 chunk *= 2;
183 bool pushed = q->push(ShenandoahMarkTask(array, chunk - 1, pow));
184 assert(pushed, "overflow queue should always succeed pushing");
185 }
186
187 int chunk_size = 1 << pow;
188
189 int from = (chunk - 1) * chunk_size;
190 int to = chunk * chunk_size;
191
192 #ifdef ASSERT
193 int len = array->length();
194 assert (0 <= from && from < len, err_msg("from is sane: %d/%d", from, len));
195 assert (0 < to && to <= len, err_msg("to is sane: %d/%d", to, len));
196 #endif
197
198 array->oop_iterate_range(cl, from, to);
199 }
200
201 inline bool ShenandoahConcurrentMark::try_queue(ShenandoahObjToScanQueue* q, ShenandoahMarkTask &task) {
202 return (q->pop_buffer(task) ||
203 q->pop_local(task) ||
204 q->pop_overflow(task));
205 }
206
207 class ShenandoahSATBBufferClosure : public SATBBufferClosure {
208 private:
209 ShenandoahObjToScanQueue* _queue;
210 ShenandoahHeap* _heap;
211 ShenandoahMarkingContext* const _mark_context;
212 public:
213 ShenandoahSATBBufferClosure(ShenandoahObjToScanQueue* q) :
214 _queue(q),
215 _heap(ShenandoahHeap::heap()),
216 _mark_context(_heap->next_marking_context())
217 {
218 }
219
220 void do_buffer(void **buffer, size_t size) {
221 if (_heap->has_forwarded_objects()) {
222 do_buffer_impl<RESOLVE>(buffer, size);
223 } else {
224 do_buffer_impl<NONE>(buffer, size);
225 }
226 }
227
228 template<UpdateRefsMode UPDATE_REFS>
229 void do_buffer_impl(void **buffer, size_t size) {
230 for (size_t i = 0; i < size; ++i) {
231 oop *p = (oop *) &buffer[i];
232 ShenandoahConcurrentMark::mark_through_ref<oop, UPDATE_REFS>(p, _heap, _queue, _mark_context);
233 }
234 }
235 };
236
237 template<class T, UpdateRefsMode UPDATE_REFS>
238 inline void ShenandoahConcurrentMark::mark_through_ref(T *p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context) {
239 ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, false /* string dedup */>(p, heap, q, mark_context, NULL);
240 }
241
242 template<class T, UpdateRefsMode UPDATE_REFS, bool STRING_DEDUP>
243 inline void ShenandoahConcurrentMark::mark_through_ref(T *p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, ShenandoahStrDedupQueue* dq) {
244 T o = oopDesc::load_heap_oop(p);
245 if (! oopDesc::is_null(o)) {
246 oop obj = oopDesc::decode_heap_oop_not_null(o);
247 switch (UPDATE_REFS) {
248 case NONE:
249 break;
250 case RESOLVE:
251 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
252 break;
253 case SIMPLE:
254 // We piggy-back reference updating to the marking tasks.
255 obj = heap->update_with_forwarded_not_null(p, obj);
256 break;
257 case CONCURRENT:
258 obj = heap->maybe_update_with_forwarded_not_null(p, obj);
259 break;
260 default:
261 ShouldNotReachHere();
262 }
263
264 // Note: Only when concurrently updating references can obj become NULL here.
265 // It happens when a mutator thread beats us by writing another value. In that
266 // case we don't need to do anything else.
267 if (UPDATE_REFS != CONCURRENT || !oopDesc::is_null(obj)) {
268 shenandoah_assert_not_forwarded(p, obj);
269 shenandoah_assert_not_in_cset_except(p, obj, heap->cancelled_gc());
270
271 if (mark_context->mark(obj)) {
272 bool pushed = q->push(ShenandoahMarkTask(obj));
273 assert(pushed, "overflow queue should always succeed pushing");
274
275 if (STRING_DEDUP && ShenandoahStringDedup::is_candidate(obj)) {
276 assert(ShenandoahStringDedup::is_enabled(), "Must be enabled");
277 assert(dq != NULL, "Dedup queue not set");
278 ShenandoahStringDedup::enqueue_candidate(obj, dq);
279 }
280 }
281
282 shenandoah_assert_marked_next(p, obj);
283 }
284 }
285 }
286
287 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP