43 size_t young_cset_length,
44 size_t optional_cset_length)
45 : _g1h(g1h),
46 _task_queue(g1h->task_queue(worker_id)),
47 _rdcq(rdcqs),
48 _ct(g1h->card_table()),
49 _closures(NULL),
50 _plab_allocator(NULL),
51 _age_table(false),
52 _tenuring_threshold(g1h->policy()->tenuring_threshold()),
53 _scanner(g1h, this),
54 _worker_id(worker_id),
55 _last_enqueued_card(SIZE_MAX),
56 _stack_trim_upper_threshold(GCDrainStackTargetSize * 2 + 1),
57 _stack_trim_lower_threshold(GCDrainStackTargetSize),
58 _trim_ticks(),
59 _surviving_young_words_base(NULL),
60 _surviving_young_words(NULL),
61 _surviving_words_length(young_cset_length + 1),
62 _old_gen_is_full(false),
63 _num_optional_regions(optional_cset_length),
64 _numa(g1h->numa()),
65 _obj_alloc_stat(NULL)
66 {
67 // We allocate number of young gen regions in the collection set plus one
68 // entries, since entry 0 keeps track of surviving bytes for non-young regions.
69 // We also add a few elements at the beginning and at the end in
70 // an attempt to eliminate cache contention
71 const size_t padding_elem_num = (DEFAULT_CACHE_LINE_SIZE / sizeof(size_t));
72 size_t array_length = padding_elem_num + _surviving_words_length + padding_elem_num;
73
74 _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length, mtGC);
75 _surviving_young_words = _surviving_young_words_base + padding_elem_num;
76 memset(_surviving_young_words, 0, _surviving_words_length * sizeof(size_t));
77
78 _plab_allocator = new G1PLABAllocator(_g1h->allocator());
79
80 // The dest for Young is used when the objects are aged enough to
81 // need to be moved to the next space.
82 _dest[G1HeapRegionAttr::Young] = G1HeapRegionAttr::Old;
182 } else {
183 obj = do_copy_to_survivor_space(region_attr, obj, m);
184 }
185 RawAccess<IS_NOT_NULL>::oop_store(p, obj);
186
187 assert(obj != NULL, "Must be");
188 if (HeapRegion::is_in_same_region(p, obj)) {
189 return;
190 }
191 HeapRegion* from = _g1h->heap_region_containing(p);
192 if (!from->is_young()) {
193 enqueue_card_if_tracked(_g1h->region_attr(obj), p, obj);
194 }
195 }
196
197 void G1ParScanThreadState::do_partial_array(PartialArrayScanTask task) {
198 oop from_obj = task.to_source_array();
199
200 assert(_g1h->is_in_reserved(from_obj), "must be in heap.");
201 assert(from_obj->is_objArray(), "must be obj array");
202 objArrayOop from_obj_array = objArrayOop(from_obj);
203 // The from-space object contains the real length.
204 int length = from_obj_array->length();
205
206 assert(from_obj->is_forwarded(), "must be forwarded");
207 oop to_obj = from_obj->forwardee();
208 assert(from_obj != to_obj, "should not be chunking self-forwarded objects");
209 objArrayOop to_obj_array = objArrayOop(to_obj);
210 // We keep track of the next start index in the length field of the
211 // to-space object.
212 int next_index = to_obj_array->length();
213 assert(0 <= next_index && next_index < length,
214 "invariant, next index: %d, length: %d", next_index, length);
215
216 int start = next_index;
217 int end = length;
218 int remainder = end - start;
219 // We'll try not to push a range that's smaller than ParGCArrayScanChunk.
220 if (remainder > 2 * ParGCArrayScanChunk) {
221 end = start + ParGCArrayScanChunk;
222 to_obj_array->set_length(end);
223 // Push the remainder before we process the range in case another
224 // worker has run out of things to do and can steal it.
225 push_on_queue(ScannerTask(PartialArrayScanTask(from_obj)));
226 } else {
227 assert(length == end, "sanity");
228 // We'll process the final range for this object. Restore the length
229 // so that the heap remains parsable in case of evacuation failure.
230 to_obj_array->set_length(end);
231 }
232
233 HeapRegion* hr = _g1h->heap_region_containing(to_obj);
234 G1ScanInYoungSetter x(&_scanner, hr->is_young());
235 // Process indexes [start,end). It will also process the header
236 // along with the first chunk (i.e., the chunk with start == 0).
237 // Note that at this point the length field of to_obj_array is not
238 // correct given that we are using it to keep track of the next
239 // start index. oop_iterate_range() (thankfully!) ignores the length
240 // field and only relies on the start / end parameters. It does
241 // however return the size of the object which will be incorrect. So
242 // we have to ignore it even if we wanted to use it.
243 to_obj_array->oop_iterate_range(&_scanner, start, end);
244 }
245
246 void G1ParScanThreadState::dispatch_task(ScannerTask task) {
247 verify_task(task);
248 if (task.is_narrow_oop_ptr()) {
249 do_oop_evac(task.to_narrow_oop_ptr());
250 } else if (task.is_oop_ptr()) {
251 do_oop_evac(task.to_oop_ptr());
252 } else {
253 do_partial_array(task.to_partial_array_task());
254 }
255 }
256
257 // Process tasks until overflow queue is empty and local queue
258 // contains no more than threshold entries. NOINLINE to prevent
259 // inlining into steal_and_trim_queue.
260 ATTRIBUTE_FLATTEN NOINLINE
261 void G1ParScanThreadState::trim_queue_to_threshold(uint threshold) {
262 ScannerTask task;
263 do {
377 }
378 return obj_ptr;
379 }
380
381 NOINLINE
382 void G1ParScanThreadState::undo_allocation(G1HeapRegionAttr dest_attr,
383 HeapWord* obj_ptr,
384 size_t word_sz,
385 uint node_index) {
386 _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
387 }
388
389 // Private inline function, for direct internal use and providing the
390 // implementation of the public not-inline function.
391 oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const region_attr,
392 oop const old,
393 markWord const old_mark) {
394 assert(region_attr.is_in_cset(),
395 "Unexpected region attr type: %s", region_attr.get_type_str());
396
397 const size_t word_sz = old->size();
398
399 uint age = 0;
400 G1HeapRegionAttr dest_attr = next_region_attr(region_attr, old_mark, age);
401 HeapRegion* const from_region = _g1h->heap_region_containing(old);
402 uint node_index = from_region->node_index();
403
404 HeapWord* obj_ptr = _plab_allocator->plab_allocate(dest_attr, word_sz, node_index);
405
406 // PLAB allocations should succeed most of the time, so we'll
407 // normally check against NULL once and that's it.
408 if (obj_ptr == NULL) {
409 obj_ptr = allocate_copy_slow(&dest_attr, old, word_sz, age, node_index);
410 if (obj_ptr == NULL) {
411 // This will either forward-to-self, or detect that someone else has
412 // installed a forwarding pointer.
413 return handle_evacuation_failure_par(old, old_mark);
414 }
415 }
416
417 assert(obj_ptr != NULL, "when we get here, allocation should have succeeded");
444
445 if (dest_attr.is_young()) {
446 if (age < markWord::max_age) {
447 age++;
448 }
449 if (old_mark.has_displaced_mark_helper()) {
450 // In this case, we have to install the mark word first,
451 // otherwise obj looks to be forwarded (the old mark word,
452 // which contains the forward pointer, was copied)
453 obj->set_mark_raw(old_mark);
454 markWord new_mark = old_mark.displaced_mark_helper().set_age(age);
455 old_mark.set_displaced_mark_helper(new_mark);
456 } else {
457 obj->set_mark_raw(old_mark.set_age(age));
458 }
459 _age_table.add(age, word_sz);
460 } else {
461 obj->set_mark_raw(old_mark);
462 }
463
464 if (G1StringDedup::is_enabled()) {
465 const bool is_from_young = region_attr.is_young();
466 const bool is_to_young = dest_attr.is_young();
467 assert(is_from_young == from_region->is_young(),
468 "sanity");
469 assert(is_to_young == _g1h->heap_region_containing(obj)->is_young(),
470 "sanity");
471 G1StringDedup::enqueue_from_evacuation(is_from_young,
472 is_to_young,
473 _worker_id,
474 obj);
475 }
476
477 if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
478 // We keep track of the next start index in the length field of
479 // the to-space object. The actual length can be found in the
480 // length field of the from-space object.
481 arrayOop(obj)->set_length(0);
482 do_partial_array(PartialArrayScanTask(old));
483 } else {
484 G1ScanInYoungSetter x(&_scanner, dest_attr.is_young());
485 obj->oop_iterate_backwards(&_scanner);
486 }
487 return obj;
488 } else {
489 _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
490 return forward_ptr;
491 }
492 }
493
494 // Public not-inline entry point.
495 ATTRIBUTE_FLATTEN
496 oop G1ParScanThreadState::copy_to_survivor_space(G1HeapRegionAttr region_attr,
497 oop old,
498 markWord old_mark) {
499 return do_copy_to_survivor_space(region_attr, old, old_mark);
500 }
501
502 G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id) {
503 assert(worker_id < _n_workers, "out of bounds access");
504 if (_states[worker_id] == NULL) {
505 _states[worker_id] =
506 new G1ParScanThreadState(_g1h, _rdcqs, worker_id, _young_cset_length, _optional_cset_length);
507 }
|
43 size_t young_cset_length,
44 size_t optional_cset_length)
45 : _g1h(g1h),
46 _task_queue(g1h->task_queue(worker_id)),
47 _rdcq(rdcqs),
48 _ct(g1h->card_table()),
49 _closures(NULL),
50 _plab_allocator(NULL),
51 _age_table(false),
52 _tenuring_threshold(g1h->policy()->tenuring_threshold()),
53 _scanner(g1h, this),
54 _worker_id(worker_id),
55 _last_enqueued_card(SIZE_MAX),
56 _stack_trim_upper_threshold(GCDrainStackTargetSize * 2 + 1),
57 _stack_trim_lower_threshold(GCDrainStackTargetSize),
58 _trim_ticks(),
59 _surviving_young_words_base(NULL),
60 _surviving_young_words(NULL),
61 _surviving_words_length(young_cset_length + 1),
62 _old_gen_is_full(false),
63 _objarray_scan_chunk_size(ParGCArrayScanChunk),
64 _objarray_length_offset_in_bytes(arrayOopDesc::length_offset_in_bytes()),
65 _num_optional_regions(optional_cset_length),
66 _numa(g1h->numa()),
67 _obj_alloc_stat(NULL)
68 {
69 // We allocate number of young gen regions in the collection set plus one
70 // entries, since entry 0 keeps track of surviving bytes for non-young regions.
71 // We also add a few elements at the beginning and at the end in
72 // an attempt to eliminate cache contention
73 const size_t padding_elem_num = (DEFAULT_CACHE_LINE_SIZE / sizeof(size_t));
74 size_t array_length = padding_elem_num + _surviving_words_length + padding_elem_num;
75
76 _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length, mtGC);
77 _surviving_young_words = _surviving_young_words_base + padding_elem_num;
78 memset(_surviving_young_words, 0, _surviving_words_length * sizeof(size_t));
79
80 _plab_allocator = new G1PLABAllocator(_g1h->allocator());
81
82 // The dest for Young is used when the objects are aged enough to
83 // need to be moved to the next space.
84 _dest[G1HeapRegionAttr::Young] = G1HeapRegionAttr::Old;
184 } else {
185 obj = do_copy_to_survivor_space(region_attr, obj, m);
186 }
187 RawAccess<IS_NOT_NULL>::oop_store(p, obj);
188
189 assert(obj != NULL, "Must be");
190 if (HeapRegion::is_in_same_region(p, obj)) {
191 return;
192 }
193 HeapRegion* from = _g1h->heap_region_containing(p);
194 if (!from->is_young()) {
195 enqueue_card_if_tracked(_g1h->region_attr(obj), p, obj);
196 }
197 }
198
199 void G1ParScanThreadState::do_partial_array(PartialArrayScanTask task) {
200 oop from_obj = task.to_source_array();
201
202 assert(_g1h->is_in_reserved(from_obj), "must be in heap.");
203 assert(from_obj->is_objArray(), "must be obj array");
204 assert(from_obj->is_forwarded(), "must be forwarded");
205
206 oop to_obj = from_obj->forwardee();
207 assert(from_obj != to_obj, "should not be chunking self-forwarded objects");
208 assert(to_obj->is_objArray(), "must be obj array");
209 objArrayOop to_array = objArrayOop(to_obj);
210
211 // The next chunk index is in the length field of the to-space object.
212 // Atomically increment by the chunk size to claim the associated chunk.
213 char* to_addr = cast_from_oop<char*>(to_array);
214 char* length_addr_raw = (to_addr + _objarray_length_offset_in_bytes);
215 volatile int* length_addr = reinterpret_cast<int*>(length_addr_raw);
216 int end = Atomic::add(length_addr, _objarray_scan_chunk_size, memory_order_relaxed);
217 #ifdef ASSERT
218 // The from-space object contains the real length.
219 int length = objArrayOop(from_obj)->length();
220 assert(end <= length, "invariant: end %d, length %d", end, length);
221 assert(((length - end) % _objarray_scan_chunk_size) == 0,
222 "invariant: end %d, length %d, chunk size %d",
223 end, length, _objarray_scan_chunk_size);
224 #endif // ASSERT
225
226 HeapRegion* hr = _g1h->heap_region_containing(to_array);
227 G1ScanInYoungSetter x(&_scanner, hr->is_young());
228 // Process claimed chunk. Note that the length field of
229 // to_obj_array is not correct. Fortunately, the iteration ignores
230 // the length and just relies on start / end. However, it does
231 // return the (incorrect) length, but we ignore it.
232 to_array->oop_iterate_range(&_scanner, end - _objarray_scan_chunk_size, end);
233 }
234
235 oop G1ParScanThreadState::start_partial_objArray(G1HeapRegionAttr dest_attr,
236 oop from_obj,
237 oop to_obj) {
238 assert(from_obj->is_objArray(), "precondition");
239 assert(from_obj->is_forwarded(), "precondition");
240 assert(from_obj->forwardee() == to_obj, "precondition");
241 assert(from_obj != to_obj, "should not be scanning self-forwarded objects");
242 assert(to_obj->is_objArray(), "precondition");
243
244 objArrayOop to_array = objArrayOop(to_obj);
245
246 int length = objArrayOop(from_obj)->length();
247 int chunks = length / _objarray_scan_chunk_size;
248 int end = length % _objarray_scan_chunk_size;
249 assert(end <= length, "invariant");
250 assert(((length - end) % _objarray_scan_chunk_size) == 0, "invariant");
251 // The value of end can be 0, either because of a 0-length array or
252 // because length is a multiple of the chunk size. Both of those
253 // are rare and handled in the normal course of the iteration, so
254 // not worth doing anything special about here.
255
256 // Set to's length to end of initial chunk. Partial tasks use that
257 // length field as the start of the next chunk to process. Must be
258 // done before enqueuing partial scan tasks, in case other threads
259 // steal any of those tasks.
260 to_array->set_length(end);
261 // Push partial scan tasks for all but the initial chunk. Pushed
262 // before processing the initial chunk to allow other workers to
263 // steal while we're processing.
264 for (int i = 0; i < chunks; ++i) {
265 push_on_queue(ScannerTask(PartialArrayScanTask(from_obj)));
266 }
267 G1ScanInYoungSetter x(&_scanner, dest_attr.is_young());
268 // Process the initial chunk. No need to process the type in the
269 // klass, as it will already be handled by processing the built-in
270 // module. The length of to_array is not correct, but fortunately
271 // the iteration ignores that length field and relies on start/end.
272 to_array->oop_iterate_range(&_scanner, 0, end);
273 return to_array;
274 }
275
276 void G1ParScanThreadState::dispatch_task(ScannerTask task) {
277 verify_task(task);
278 if (task.is_narrow_oop_ptr()) {
279 do_oop_evac(task.to_narrow_oop_ptr());
280 } else if (task.is_oop_ptr()) {
281 do_oop_evac(task.to_oop_ptr());
282 } else {
283 do_partial_array(task.to_partial_array_task());
284 }
285 }
286
287 // Process tasks until overflow queue is empty and local queue
288 // contains no more than threshold entries. NOINLINE to prevent
289 // inlining into steal_and_trim_queue.
290 ATTRIBUTE_FLATTEN NOINLINE
291 void G1ParScanThreadState::trim_queue_to_threshold(uint threshold) {
292 ScannerTask task;
293 do {
407 }
408 return obj_ptr;
409 }
410
411 NOINLINE
412 void G1ParScanThreadState::undo_allocation(G1HeapRegionAttr dest_attr,
413 HeapWord* obj_ptr,
414 size_t word_sz,
415 uint node_index) {
416 _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
417 }
418
419 // Private inline function, for direct internal use and providing the
420 // implementation of the public not-inline function.
421 oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const region_attr,
422 oop const old,
423 markWord const old_mark) {
424 assert(region_attr.is_in_cset(),
425 "Unexpected region attr type: %s", region_attr.get_type_str());
426
427 // Get the klass once. We'll need it again later, and this avoids
428 // re-decoding when it's compressed.
429 Klass* klass = old->klass();
430 const size_t word_sz = old->size_given_klass(klass);
431
432 uint age = 0;
433 G1HeapRegionAttr dest_attr = next_region_attr(region_attr, old_mark, age);
434 HeapRegion* const from_region = _g1h->heap_region_containing(old);
435 uint node_index = from_region->node_index();
436
437 HeapWord* obj_ptr = _plab_allocator->plab_allocate(dest_attr, word_sz, node_index);
438
439 // PLAB allocations should succeed most of the time, so we'll
440 // normally check against NULL once and that's it.
441 if (obj_ptr == NULL) {
442 obj_ptr = allocate_copy_slow(&dest_attr, old, word_sz, age, node_index);
443 if (obj_ptr == NULL) {
444 // This will either forward-to-self, or detect that someone else has
445 // installed a forwarding pointer.
446 return handle_evacuation_failure_par(old, old_mark);
447 }
448 }
449
450 assert(obj_ptr != NULL, "when we get here, allocation should have succeeded");
477
478 if (dest_attr.is_young()) {
479 if (age < markWord::max_age) {
480 age++;
481 }
482 if (old_mark.has_displaced_mark_helper()) {
483 // In this case, we have to install the mark word first,
484 // otherwise obj looks to be forwarded (the old mark word,
485 // which contains the forward pointer, was copied)
486 obj->set_mark_raw(old_mark);
487 markWord new_mark = old_mark.displaced_mark_helper().set_age(age);
488 old_mark.set_displaced_mark_helper(new_mark);
489 } else {
490 obj->set_mark_raw(old_mark.set_age(age));
491 }
492 _age_table.add(age, word_sz);
493 } else {
494 obj->set_mark_raw(old_mark);
495 }
496
497 // Most objects are not arrays, so do one array check rather than both
498 // typeArray and objArray checks for each object.
499 if (klass->is_array_klass()) {
500 if (klass->is_typeArray_klass()) {
501 // Nothing needs to be done for typeArrays. Body doesn't contain
502 // any oops to scan, and the type in the klass will already be handled
503 // by processing the built-in module.
504 return obj;
505 } else if (klass->is_objArray_klass()) {
506 // Do special handling for objArray.
507 return start_partial_objArray(dest_attr, old, obj);
508 }
509 // Not a special array, so fall through to generic handling.
510 }
511
512 if (G1StringDedup::is_enabled() && (klass == SystemDictionary::String_klass())) {
513 const bool is_from_young = region_attr.is_young();
514 const bool is_to_young = dest_attr.is_young();
515 assert(is_from_young == from_region->is_young(),
516 "sanity");
517 assert(is_to_young == _g1h->heap_region_containing(obj)->is_young(),
518 "sanity");
519 G1StringDedup::enqueue_from_evacuation(is_from_young,
520 is_to_young,
521 _worker_id,
522 obj);
523 }
524
525 G1ScanInYoungSetter x(&_scanner, dest_attr.is_young());
526 obj->oop_iterate_backwards(&_scanner);
527 return obj;
528
529 } else {
530 _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
531 return forward_ptr;
532 }
533 }
534
535 // Public not-inline entry point.
536 ATTRIBUTE_FLATTEN
537 oop G1ParScanThreadState::copy_to_survivor_space(G1HeapRegionAttr region_attr,
538 oop old,
539 markWord old_mark) {
540 return do_copy_to_survivor_space(region_attr, old, old_mark);
541 }
542
543 G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id) {
544 assert(worker_id < _n_workers, "out of bounds access");
545 if (_states[worker_id] == NULL) {
546 _states[worker_id] =
547 new G1ParScanThreadState(_g1h, _rdcqs, worker_id, _young_cset_length, _optional_cset_length);
548 }
|