47 }
48 assert(! oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must be evacuated");
49 // Update reference.
50 _heap->atomic_compare_exchange_oop(forw, p, obj);
51 obj = forw;
52 }
53
54 if (!_heap->is_marked_next(obj) && _heap->mark_next(obj)) {
55 bool succeeded = queue->push(ShenandoahMarkTask(obj));
56 assert(succeeded, "must succeed to push to task queue");
57
58 if (STRING_DEDUP && ShenandoahStringDedup::is_candidate(obj)) {
59 assert(ShenandoahStringDedup::is_enabled(), "Must be enabled");
60 assert(dq != NULL, "Dedup queue not set");
61 ShenandoahStringDedup::enqueue_candidate(obj, dq);
62 }
63 }
64 }
65 }
66
67 template <class T>
68 void ShenandoahTraversalGC::do_task(ShenandoahObjToScanQueue* q, T* cl, jushort* live_data, ShenandoahMarkTask* task) {
69 oop obj = task->obj();
70
71 shenandoah_assert_not_forwarded_except(NULL, obj, _heap->cancelled_concgc());
72 shenandoah_assert_marked_next(NULL, obj);
73 shenandoah_assert_not_in_cset_except(NULL, obj, _heap->cancelled_concgc());
74
75 if (task->is_not_chunked()) {
76 count_liveness(live_data, obj);
77 if (obj->is_instance()) {
78 // Case 1: Normal oop, process as usual.
79 obj->oop_iterate(cl);
80 } else if (obj->is_objArray()) {
81 // Case 2: Object array instance and no chunk is set. Must be the first
82 // time we visit it, start the chunked processing.
83 do_chunked_array_start(q, cl, obj);
84 } else {
85 // Case 3: Primitive array. Do nothing, no oops there. We use the same
86 // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using:
87 // We skip iterating over the klass pointer since we know that
88 // Universe::TypeArrayKlass never moves.
89 assert (obj->is_typeArray(), "should be type array");
90 }
91 } else {
92 // Case 4: Array chunk, has sensible chunk id. Process it.
93 do_chunked_array(q, cl, obj, task->chunk(), task->pow());
94 }
95 }
96
97 inline void ShenandoahTraversalGC::count_liveness(jushort* live_data, oop obj) {
98 size_t region_idx = _heap->heap_region_index_containing(obj);
99 jushort cur = live_data[region_idx];
100 int size = obj->size() + BrooksPointer::word_size();
101 int max = (1 << (sizeof(jushort) * 8)) - 1;
102 if (size >= max) {
103 // too big, add to region data directly
104 _heap->regions()->get(region_idx)->increase_live_data_words(size);
105 } else {
106 int new_val = cur + size;
107 if (new_val >= max) {
108 // overflow, flush to region data
109 _heap->regions()->get(region_idx)->increase_live_data_words(new_val);
110 live_data[region_idx] = 0;
111 } else {
112 // still good, remember in locals
113 live_data[region_idx] = (jushort) new_val;
114 }
115 }
116 }
117
118 template <class T>
119 inline void ShenandoahTraversalGC::do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop obj) {
120 assert(obj->is_objArray(), "expect object array");
121 objArrayOop array = objArrayOop(obj);
122 int len = array->length();
123
124 if (len <= (int) ObjArrayMarkingStride*2) {
125 // A few slices only, process directly
126 array->oop_iterate_range(cl, 0, len);
127 } else {
128 int bits = log2_long(len);
129 // Compensate for non-power-of-two arrays, cover the array in excess:
130 if (len != (1 << bits)) bits++;
131
132 // Only allow full chunks on the queue. This frees do_chunked_array() from checking from/to
133 // boundaries against array->length(), touching the array header on every chunk.
134 //
135 // To do this, we cut the prefix in full-sized chunks, and submit them on the queue.
136 // If the array is not divided in chunk sizes, then there would be an irregular tail,
137 // which we will process separately.
138
139 int last_idx = 0;
140
141 int chunk = 1;
142 int pow = bits;
143
144 // Handle overflow
145 if (pow >= 31) {
146 assert (pow == 31, "sanity");
147 pow--;
148 chunk = 2;
149 last_idx = (1 << pow);
150 bool pushed = q->push(ShenandoahMarkTask(array, 1, pow));
151 assert(pushed, "overflow queue should always succeed pushing");
152 }
153
154 // Split out tasks, as suggested in ObjArrayChunkedTask docs. Record the last
155 // successful right boundary to figure out the irregular tail.
156 while ((1 << pow) > (int)ObjArrayMarkingStride &&
157 (chunk*2 < ShenandoahMarkTask::chunk_size())) {
158 pow--;
159 int left_chunk = chunk*2 - 1;
160 int right_chunk = chunk*2;
161 int left_chunk_end = left_chunk * (1 << pow);
162 if (left_chunk_end < len) {
163 bool pushed = q->push(ShenandoahMarkTask(array, left_chunk, pow));
164 assert(pushed, "overflow queue should always succeed pushing");
165 chunk = right_chunk;
166 last_idx = left_chunk_end;
167 } else {
168 chunk = left_chunk;
169 }
170 }
171
172 // Process the irregular tail, if present
173 int from = last_idx;
174 if (from < len) {
175 array->oop_iterate_range(cl, from, len);
176 }
177 }
178 }
179
180 template <class T>
181 inline void ShenandoahTraversalGC::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop obj, int chunk, int pow) {
182 assert(obj->is_objArray(), "expect object array");
183 objArrayOop array = objArrayOop(obj);
184
185 assert (ObjArrayMarkingStride > 0, "sanity");
186
187 // Split out tasks, as suggested in ObjArrayChunkedTask docs. Avoid pushing tasks that
188 // are known to start beyond the array.
189 while ((1 << pow) > (int)ObjArrayMarkingStride && (chunk*2 < ShenandoahMarkTask::chunk_size())) {
190 pow--;
191 chunk *= 2;
192 bool pushed = q->push(ShenandoahMarkTask(array, chunk - 1, pow));
193 assert(pushed, "overflow queue should always succeed pushing");
194 }
195
196 int chunk_size = 1 << pow;
197
198 int from = (chunk - 1) * chunk_size;
199 int to = chunk * chunk_size;
200
201 #ifdef ASSERT
202 int len = array->length();
203 assert (0 <= from && from < len, "from is sane: %d/%d", from, len);
204 assert (0 < to && to <= len, "to is sane: %d/%d", to, len);
205 #endif
206
207 array->oop_iterate_range(cl, from, to);
208 }
209
210 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_INLINE_HPP
|
47 }
48 assert(! oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must be evacuated");
49 // Update reference.
50 _heap->atomic_compare_exchange_oop(forw, p, obj);
51 obj = forw;
52 }
53
54 if (!_heap->is_marked_next(obj) && _heap->mark_next(obj)) {
55 bool succeeded = queue->push(ShenandoahMarkTask(obj));
56 assert(succeeded, "must succeed to push to task queue");
57
58 if (STRING_DEDUP && ShenandoahStringDedup::is_candidate(obj)) {
59 assert(ShenandoahStringDedup::is_enabled(), "Must be enabled");
60 assert(dq != NULL, "Dedup queue not set");
61 ShenandoahStringDedup::enqueue_candidate(obj, dq);
62 }
63 }
64 }
65 }
66
67 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_INLINE_HPP
|