Print this page
rev 3640 : 7200261: G1: Liveness counting inconsistencies during marking verification
Summary: The clipping code in the routine that sets the bits for a range of cards, in the liveness accounting verification code was incorrect. It set all the bits in the card bitmap from the given starting index which would lead to spurious marking verification failures.
Reviewed-by:
* * *
[mq]: code-review-comments
Split |
Close |
Expand all |
Collapse all |
--- old/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp
+++ new/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp
1 1 /*
2 2 * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
3 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 4 *
5 5 * This code is free software; you can redistribute it and/or modify it
6 6 * under the terms of the GNU General Public License version 2 only, as
7 7 * published by the Free Software Foundation.
8 8 *
9 9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 12 * version 2 for more details (a copy is included in the LICENSE file that
13 13 * accompanied this code).
14 14 *
15 15 * You should have received a copy of the GNU General Public License version
16 16 * 2 along with this work; if not, write to the Free Software Foundation,
17 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 18 *
19 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 20 * or visit www.oracle.com if you need additional information or have any
↓ open down ↓ |
20 lines elided |
↑ open up ↑ |
21 21 * questions.
22 22 *
23 23 */
24 24
25 25 #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP
26 26 #define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP
27 27
28 28 #include "gc_implementation/g1/concurrentMark.hpp"
29 29 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
30 30
31 +// Utility routine to set an exclusive range of cards on the given
32 +// card liveness bitmap
33 +inline void ConcurrentMark::set_card_bitmap_range(BitMap* card_bm,
34 + BitMap::idx_t start_idx,
35 + BitMap::idx_t end_idx,
36 + bool is_par) {
37 +
38 + // Set the exclusive bit range [start_idx, end_idx).
39 + assert((end_idx - start_idx) > 0, "at least one card");
40 + assert(end_idx <= card_bm->size(), "sanity");
41 +
42 + // Silently clip the end index
43 + end_idx = MIN2(end_idx, card_bm->size());
44 +
45 + // For small ranges use a simple loop; otherwise use set_range or
46 + // use par_at_put_range (if parallel). The range is made up of the
47 + // cards that are spanned by an object/mem region so 8 cards will
48 + // allow up to object sizes up to 4K to be handled using the loop.
49 + if ((end_idx - start_idx) <= 8) {
50 + for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
51 + if (is_par) {
52 + card_bm->par_set_bit(i);
53 + } else {
54 + card_bm->set_bit(i);
55 + }
56 + }
57 + } else {
58 + // Note BitMap::par_at_put_range() and BitMap::set_range() are exclusive.
59 + if (is_par) {
60 + card_bm->par_at_put_range(start_idx, end_idx, true);
61 + } else {
62 + card_bm->set_range(start_idx, end_idx);
63 + }
64 + }
65 +}
66 +
31 67 // Returns the index in the liveness accounting card bitmap
32 68 // for the given address
33 69 inline BitMap::idx_t ConcurrentMark::card_bitmap_index_for(HeapWord* addr) {
34 70 // Below, the term "card num" means the result of shifting an address
35 71 // by the card shift -- address 0 corresponds to card number 0. One
36 72 // must subtract the card num of the bottom of the heap to obtain a
37 73 // card table index.
38 -
39 74 intptr_t card_num = intptr_t(uintptr_t(addr) >> CardTableModRefBS::card_shift);
40 75 return card_num - heap_bottom_card_num();
41 76 }
42 77
43 78 // Counts the given memory region in the given task/worker
44 79 // counting data structures.
45 80 inline void ConcurrentMark::count_region(MemRegion mr, HeapRegion* hr,
46 81 size_t* marked_bytes_array,
47 82 BitMap* task_card_bm) {
48 83 G1CollectedHeap* g1h = _g1h;
84 + CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set());
85 +
49 86 HeapWord* start = mr.start();
50 - HeapWord* last = mr.last();
87 + HeapWord* end = mr.end();
51 88 size_t region_size_bytes = mr.byte_size();
52 89 uint index = hr->hrs_index();
53 90
54 91 assert(!hr->continuesHumongous(), "should not be HC region");
55 92 assert(hr == g1h->heap_region_containing(start), "sanity");
56 93 assert(hr == g1h->heap_region_containing(mr.last()), "sanity");
57 94 assert(marked_bytes_array != NULL, "pre-condition");
58 95 assert(task_card_bm != NULL, "pre-condition");
59 96
60 97 // Add to the task local marked bytes for this region.
61 98 marked_bytes_array[index] += region_size_bytes;
62 99
63 100 BitMap::idx_t start_idx = card_bitmap_index_for(start);
64 - BitMap::idx_t last_idx = card_bitmap_index_for(last);
101 + BitMap::idx_t end_idx = card_bitmap_index_for(end);
65 102
66 - // The card bitmap is task/worker specific => no need to use 'par' routines.
67 - // Set bits in the inclusive bit range [start_idx, last_idx].
68 - //
69 - // For small ranges use a simple loop; otherwise use set_range
70 - // The range are the cards that are spanned by the object/region
71 - // so 8 cards will allow objects/regions up to 4K to be handled
72 - // using the loop.
73 - if ((last_idx - start_idx) <= 8) {
74 - for (BitMap::idx_t i = start_idx; i <= last_idx; i += 1) {
75 - task_card_bm->set_bit(i);
76 - }
77 - } else {
78 - assert(last_idx < task_card_bm->size(), "sanity");
79 - // Note: BitMap::set_range() is exclusive.
80 - task_card_bm->set_range(start_idx, last_idx+1);
81 - }
103 + // Note: if we're looking at a region that coincides with the end
104 + // of the heap - end could actually be outside the heap and end_idx
105 + // correspond to a card that is also outside the heap.
106 + if (g1h->is_in_g1_reserved(end) && !ct_bs->is_card_aligned(end)) {
107 + // end of region is not card aligned - incremement to cover
108 + // all the cards spanned by the region.
109 + end_idx += 1;
110 + }
111 + // The card bitmap is task/worker specific => no need to use
112 + // the 'par' BitMap routines.
113 + // Set bits in the exclusive bit range [start_idx, end_idx).
114 + set_card_bitmap_range(task_card_bm, start_idx, end_idx, false /* is_par */);
82 115 }
83 116
84 117 // Counts the given memory region in the task/worker counting
85 118 // data structures for the given worker id.
86 119 inline void ConcurrentMark::count_region(MemRegion mr,
87 120 HeapRegion* hr,
88 121 uint worker_id) {
89 122 size_t* marked_bytes_array = count_marked_bytes_array_for(worker_id);
90 123 BitMap* task_card_bm = count_card_bitmap_for(worker_id);
91 124 count_region(mr, hr, marked_bytes_array, task_card_bm);
92 125 }
93 126
94 127 // Counts the given memory region, which may be a single object, in the
95 128 // task/worker counting data structures for the given worker id.
96 129 inline void ConcurrentMark::count_region(MemRegion mr, uint worker_id) {
97 130 HeapWord* addr = mr.start();
98 131 HeapRegion* hr = _g1h->heap_region_containing_raw(addr);
99 132 count_region(mr, hr, worker_id);
100 133 }
101 134
102 135 // Counts the given object in the given task/worker counting data structures.
103 136 inline void ConcurrentMark::count_object(oop obj,
104 137 HeapRegion* hr,
105 138 size_t* marked_bytes_array,
106 139 BitMap* task_card_bm) {
107 140 MemRegion mr((HeapWord*)obj, obj->size());
108 141 count_region(mr, hr, marked_bytes_array, task_card_bm);
109 142 }
110 143
111 144 // Counts the given object in the task/worker counting data
112 145 // structures for the given worker id.
113 146 inline void ConcurrentMark::count_object(oop obj,
114 147 HeapRegion* hr,
115 148 uint worker_id) {
116 149 size_t* marked_bytes_array = count_marked_bytes_array_for(worker_id);
117 150 BitMap* task_card_bm = count_card_bitmap_for(worker_id);
118 151 HeapWord* addr = (HeapWord*) obj;
119 152 count_object(obj, hr, marked_bytes_array, task_card_bm);
120 153 }
121 154
122 155 // Attempts to mark the given object and, if successful, counts
123 156 // the object in the given task/worker counting structures.
124 157 inline bool ConcurrentMark::par_mark_and_count(oop obj,
125 158 HeapRegion* hr,
126 159 size_t* marked_bytes_array,
127 160 BitMap* task_card_bm) {
128 161 HeapWord* addr = (HeapWord*)obj;
129 162 if (_nextMarkBitMap->parMark(addr)) {
130 163 // Update the task specific count data for the object.
131 164 count_object(obj, hr, marked_bytes_array, task_card_bm);
132 165 return true;
133 166 }
134 167 return false;
135 168 }
136 169
137 170 // Attempts to mark the given object and, if successful, counts
138 171 // the object in the task/worker counting structures for the
139 172 // given worker id.
140 173 inline bool ConcurrentMark::par_mark_and_count(oop obj,
141 174 size_t word_size,
142 175 HeapRegion* hr,
143 176 uint worker_id) {
144 177 HeapWord* addr = (HeapWord*)obj;
145 178 if (_nextMarkBitMap->parMark(addr)) {
146 179 MemRegion mr(addr, word_size);
147 180 count_region(mr, hr, worker_id);
148 181 return true;
149 182 }
150 183 return false;
151 184 }
152 185
153 186 // Attempts to mark the given object and, if successful, counts
154 187 // the object in the task/worker counting structures for the
155 188 // given worker id.
156 189 inline bool ConcurrentMark::par_mark_and_count(oop obj,
157 190 HeapRegion* hr,
158 191 uint worker_id) {
159 192 HeapWord* addr = (HeapWord*)obj;
160 193 if (_nextMarkBitMap->parMark(addr)) {
161 194 // Update the task specific count data for the object.
162 195 count_object(obj, hr, worker_id);
163 196 return true;
164 197 }
165 198 return false;
166 199 }
167 200
168 201 // As above - but we don't know the heap region containing the
169 202 // object and so have to supply it.
170 203 inline bool ConcurrentMark::par_mark_and_count(oop obj, uint worker_id) {
171 204 HeapWord* addr = (HeapWord*)obj;
172 205 HeapRegion* hr = _g1h->heap_region_containing_raw(addr);
173 206 return par_mark_and_count(obj, hr, worker_id);
174 207 }
175 208
176 209 // Similar to the above routine but we already know the size, in words, of
177 210 // the object that we wish to mark/count
178 211 inline bool ConcurrentMark::par_mark_and_count(oop obj,
179 212 size_t word_size,
180 213 uint worker_id) {
181 214 HeapWord* addr = (HeapWord*)obj;
182 215 if (_nextMarkBitMap->parMark(addr)) {
183 216 // Update the task specific count data for the object.
184 217 MemRegion mr(addr, word_size);
185 218 count_region(mr, worker_id);
186 219 return true;
187 220 }
188 221 return false;
189 222 }
190 223
191 224 // Unconditionally mark the given object, and unconditinally count
192 225 // the object in the counting structures for worker id 0.
193 226 // Should *not* be called from parallel code.
194 227 inline bool ConcurrentMark::mark_and_count(oop obj, HeapRegion* hr) {
195 228 HeapWord* addr = (HeapWord*)obj;
196 229 _nextMarkBitMap->mark(addr);
197 230 // Update the task specific count data for the object.
198 231 count_object(obj, hr, 0 /* worker_id */);
199 232 return true;
200 233 }
201 234
202 235 // As above - but we don't have the heap region containing the
203 236 // object, so we have to supply it.
204 237 inline bool ConcurrentMark::mark_and_count(oop obj) {
205 238 HeapWord* addr = (HeapWord*)obj;
206 239 HeapRegion* hr = _g1h->heap_region_containing_raw(addr);
207 240 return mark_and_count(obj, hr);
208 241 }
209 242
210 243 inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
211 244 HeapWord* start_addr = MAX2(startWord(), mr.start());
212 245 HeapWord* end_addr = MIN2(endWord(), mr.end());
213 246
214 247 if (end_addr > start_addr) {
215 248 // Right-open interval [start-offset, end-offset).
216 249 BitMap::idx_t start_offset = heapWordToOffset(start_addr);
217 250 BitMap::idx_t end_offset = heapWordToOffset(end_addr);
218 251
219 252 start_offset = _bm.get_next_one_offset(start_offset, end_offset);
220 253 while (start_offset < end_offset) {
221 254 HeapWord* obj_addr = offsetToHeapWord(start_offset);
222 255 oop obj = (oop) obj_addr;
223 256 if (!cl->do_bit(start_offset)) {
224 257 return false;
225 258 }
226 259 HeapWord* next_addr = MIN2(obj_addr + obj->size(), end_addr);
227 260 BitMap::idx_t next_offset = heapWordToOffset(next_addr);
228 261 start_offset = _bm.get_next_one_offset(next_offset, end_offset);
229 262 }
230 263 }
231 264 return true;
232 265 }
233 266
234 267 inline bool CMBitMapRO::iterate(BitMapClosure* cl) {
235 268 MemRegion mr(startWord(), sizeInWords());
236 269 return iterate(cl, mr);
237 270 }
238 271
239 272 inline void CMTask::push(oop obj) {
240 273 HeapWord* objAddr = (HeapWord*) obj;
241 274 assert(_g1h->is_in_g1_reserved(objAddr), "invariant");
242 275 assert(!_g1h->is_on_master_free_list(
243 276 _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant");
244 277 assert(!_g1h->is_obj_ill(obj), "invariant");
245 278 assert(_nextMarkBitMap->isMarked(objAddr), "invariant");
246 279
247 280 if (_cm->verbose_high()) {
248 281 gclog_or_tty->print_cr("[%d] pushing "PTR_FORMAT, _task_id, (void*) obj);
249 282 }
250 283
251 284 if (!_task_queue->push(obj)) {
252 285 // The local task queue looks full. We need to push some entries
253 286 // to the global stack.
254 287
255 288 if (_cm->verbose_medium()) {
256 289 gclog_or_tty->print_cr("[%d] task queue overflow, "
257 290 "moving entries to the global stack",
258 291 _task_id);
259 292 }
260 293 move_entries_to_global_stack();
261 294
262 295 // this should succeed since, even if we overflow the global
263 296 // stack, we should have definitely removed some entries from the
264 297 // local queue. So, there must be space on it.
265 298 bool success = _task_queue->push(obj);
266 299 assert(success, "invariant");
267 300 }
268 301
269 302 statsOnly( int tmp_size = _task_queue->size();
270 303 if (tmp_size > _local_max_size) {
271 304 _local_max_size = tmp_size;
272 305 }
273 306 ++_local_pushes );
274 307 }
275 308
276 309 // This determines whether the method below will check both the local
277 310 // and global fingers when determining whether to push on the stack a
278 311 // gray object (value 1) or whether it will only check the global one
279 312 // (value 0). The tradeoffs are that the former will be a bit more
280 313 // accurate and possibly push less on the stack, but it might also be
281 314 // a little bit slower.
282 315
283 316 #define _CHECK_BOTH_FINGERS_ 1
284 317
285 318 inline void CMTask::deal_with_reference(oop obj) {
286 319 if (_cm->verbose_high()) {
287 320 gclog_or_tty->print_cr("[%d] we're dealing with reference = "PTR_FORMAT,
288 321 _task_id, (void*) obj);
289 322 }
290 323
291 324 ++_refs_reached;
292 325
293 326 HeapWord* objAddr = (HeapWord*) obj;
294 327 assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
295 328 if (_g1h->is_in_g1_reserved(objAddr)) {
296 329 assert(obj != NULL, "null check is implicit");
297 330 if (!_nextMarkBitMap->isMarked(objAddr)) {
298 331 // Only get the containing region if the object is not marked on the
299 332 // bitmap (otherwise, it's a waste of time since we won't do
300 333 // anything with it).
301 334 HeapRegion* hr = _g1h->heap_region_containing_raw(obj);
302 335 if (!hr->obj_allocated_since_next_marking(obj)) {
303 336 if (_cm->verbose_high()) {
304 337 gclog_or_tty->print_cr("[%d] "PTR_FORMAT" is not considered marked",
305 338 _task_id, (void*) obj);
306 339 }
307 340
308 341 // we need to mark it first
309 342 if (_cm->par_mark_and_count(obj, hr, _marked_bytes_array, _card_bm)) {
310 343 // No OrderAccess:store_load() is needed. It is implicit in the
311 344 // CAS done in CMBitMap::parMark() call in the routine above.
312 345 HeapWord* global_finger = _cm->finger();
313 346
314 347 #if _CHECK_BOTH_FINGERS_
315 348 // we will check both the local and global fingers
316 349
317 350 if (_finger != NULL && objAddr < _finger) {
318 351 if (_cm->verbose_high()) {
319 352 gclog_or_tty->print_cr("[%d] below the local finger ("PTR_FORMAT"), "
320 353 "pushing it", _task_id, _finger);
321 354 }
322 355 push(obj);
323 356 } else if (_curr_region != NULL && objAddr < _region_limit) {
324 357 // do nothing
325 358 } else if (objAddr < global_finger) {
326 359 // Notice that the global finger might be moving forward
327 360 // concurrently. This is not a problem. In the worst case, we
328 361 // mark the object while it is above the global finger and, by
329 362 // the time we read the global finger, it has moved forward
330 363 // passed this object. In this case, the object will probably
331 364 // be visited when a task is scanning the region and will also
332 365 // be pushed on the stack. So, some duplicate work, but no
333 366 // correctness problems.
334 367
335 368 if (_cm->verbose_high()) {
336 369 gclog_or_tty->print_cr("[%d] below the global finger "
337 370 "("PTR_FORMAT"), pushing it",
338 371 _task_id, global_finger);
339 372 }
340 373 push(obj);
341 374 } else {
342 375 // do nothing
343 376 }
344 377 #else // _CHECK_BOTH_FINGERS_
345 378 // we will only check the global finger
346 379
347 380 if (objAddr < global_finger) {
348 381 // see long comment above
349 382
350 383 if (_cm->verbose_high()) {
351 384 gclog_or_tty->print_cr("[%d] below the global finger "
352 385 "("PTR_FORMAT"), pushing it",
353 386 _task_id, global_finger);
354 387 }
355 388 push(obj);
356 389 }
357 390 #endif // _CHECK_BOTH_FINGERS_
358 391 }
359 392 }
360 393 }
361 394 }
362 395 }
363 396
364 397 inline void ConcurrentMark::markPrev(oop p) {
365 398 assert(!_prevMarkBitMap->isMarked((HeapWord*) p), "sanity");
366 399 // Note we are overriding the read-only view of the prev map here, via
367 400 // the cast.
368 401 ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*) p);
369 402 }
370 403
371 404 inline void ConcurrentMark::grayRoot(oop obj, size_t word_size,
372 405 uint worker_id, HeapRegion* hr) {
373 406 assert(obj != NULL, "pre-condition");
374 407 HeapWord* addr = (HeapWord*) obj;
375 408 if (hr == NULL) {
376 409 hr = _g1h->heap_region_containing_raw(addr);
377 410 } else {
378 411 assert(hr->is_in(addr), "pre-condition");
379 412 }
380 413 assert(hr != NULL, "sanity");
381 414 // Given that we're looking for a region that contains an object
382 415 // header it's impossible to get back a HC region.
383 416 assert(!hr->continuesHumongous(), "sanity");
384 417
385 418 // We cannot assert that word_size == obj->size() given that obj
386 419 // might not be in a consistent state (another thread might be in
387 420 // the process of copying it). So the best thing we can do is to
388 421 // assert that word_size is under an upper bound which is its
389 422 // containing region's capacity.
390 423 assert(word_size * HeapWordSize <= hr->capacity(),
391 424 err_msg("size: "SIZE_FORMAT" capacity: "SIZE_FORMAT" "HR_FORMAT,
392 425 word_size * HeapWordSize, hr->capacity(),
393 426 HR_FORMAT_PARAMS(hr)));
394 427
395 428 if (addr < hr->next_top_at_mark_start()) {
396 429 if (!_nextMarkBitMap->isMarked(addr)) {
397 430 par_mark_and_count(obj, word_size, hr, worker_id);
398 431 }
399 432 }
400 433 }
401 434
402 435 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP
↓ open down ↓ |
311 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX