4037 // would point into middle of the filler object.
4038 // The current approach is to not coalesce and leave the BOT contents intact.
4039 // </original comment>
4040 //
4041 // We now reset the BOT when we start the object iteration over the
4042 // region and refine its entries for every object we come across. So
4043 // the above comment is not really relevant and we should be able
4044 // to coalesce dead objects if we want to.
4045 void do_object(oop obj) {
4046 HeapWord* obj_addr = (HeapWord*) obj;
4047 assert(_hr->is_in(obj_addr), "sanity");
4048 size_t obj_size = obj->size();
4049 _hr->update_bot_for_object(obj_addr, obj_size);
4050 if (obj->is_forwarded() && obj->forwardee() == obj) {
4051 // The object failed to move.
4052 assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs.");
4053 _cm->markPrev(obj);
4054 assert(_cm->isPrevMarked(obj), "Should be marked!");
4055 _prev_marked_bytes += (obj_size * HeapWordSize);
4056 if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) {
4057 _cm->markAndGrayObjectIfNecessary(obj);
4058 }
4059 obj->set_mark(markOopDesc::prototype());
4060 // While we were processing RSet buffers during the
4061 // collection, we actually didn't scan any cards on the
4062 // collection set, since we didn't want to update remebered
4063 // sets with entries that point into the collection set, given
4064 // that live objects fromthe collection set are about to move
4065 // and such entries will be stale very soon. This change also
4066 // dealt with a reliability issue which involved scanning a
4067 // card in the collection set and coming across an array that
4068 // was being chunked and looking malformed. The problem is
4069 // that, if evacuation fails, we might have remembered set
4070 // entries missing given that we skipped cards on the
4071 // collection set. So, we'll recreate such entries now.
4072 obj->oop_iterate(_cl);
4073 assert(_cm->isPrevMarked(obj), "Should be marked!");
4074 } else {
4075 // The object has been either evacuated or is dead. Fill it with a
4076 // dummy object.
4077 MemRegion mr((HeapWord*)obj, obj_size);
4156 }
4157 }
4158
4159 void G1CollectedHeap::push_on_evac_failure_scan_stack(oop obj) {
4160 _evac_failure_scan_stack->push(obj);
4161 }
4162
4163 void G1CollectedHeap::drain_evac_failure_scan_stack() {
4164 assert(_evac_failure_scan_stack != NULL, "precondition");
4165
4166 while (_evac_failure_scan_stack->length() > 0) {
4167 oop obj = _evac_failure_scan_stack->pop();
4168 _evac_failure_closure->set_region(heap_region_containing(obj));
4169 obj->oop_iterate_backwards(_evac_failure_closure);
4170 }
4171 }
4172
4173 oop
4174 G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl,
4175 oop old,
4176 bool should_mark_root) {
4177 assert(obj_in_cs(old),
4178 err_msg("obj: "PTR_FORMAT" should still be in the CSet",
4179 (HeapWord*) old));
4180 markOop m = old->mark();
4181 oop forward_ptr = old->forward_to_atomic(old);
4182 if (forward_ptr == NULL) {
4183 // Forward-to-self succeeded.
4184
4185 // should_mark_root will be true when this routine is called
4186 // from a root scanning closure during an initial mark pause.
4187 // In this case the thread that succeeds in self-forwarding the
4188 // object is also responsible for marking the object.
4189 if (should_mark_root) {
4190 assert(!oopDesc::is_null(old), "shouldn't be");
4191 _cm->grayRoot(old);
4192 }
4193
4194 if (_evac_failure_closure != cl) {
4195 MutexLockerEx x(EvacFailureStack_lock, Mutex::_no_safepoint_check_flag);
4196 assert(!_drain_in_progress,
4197 "Should only be true while someone holds the lock.");
4198 // Set the global evac-failure closure to the current thread's.
4199 assert(_evac_failure_closure == NULL, "Or locking has failed.");
4200 set_evac_failure_closure(cl);
4201 // Now do the common part.
4202 handle_evacuation_failure_common(old, m);
4203 // Reset to NULL.
4204 set_evac_failure_closure(NULL);
4205 } else {
4206 // The lock is already held, and this is recursive.
4207 assert(_drain_in_progress, "This should only be the recursive case.");
4208 handle_evacuation_failure_common(old, m);
4209 }
4210 return old;
4211 } else {
4277 } else {
4278 // Let's try to allocate in the survivors in case we can fit the
4279 // object there.
4280 return survivor_attempt_allocation(word_size);
4281 }
4282 }
4283
4284 ShouldNotReachHere();
4285 // Trying to keep some compilers happy.
4286 return NULL;
4287 }
4288
4289 #ifndef PRODUCT
4290 bool GCLabBitMapClosure::do_bit(size_t offset) {
4291 HeapWord* addr = _bitmap->offsetToHeapWord(offset);
4292 guarantee(_cm->isMarked(oop(addr)), "it should be!");
4293 return true;
4294 }
4295 #endif // PRODUCT
4296
4297 G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size) :
4298 ParGCAllocBuffer(gclab_word_size),
4299 _should_mark_objects(false),
4300 _bitmap(G1CollectedHeap::heap()->reserved_region().start(), gclab_word_size),
4301 _retired(false)
4302 {
4303 //_should_mark_objects is set to true when G1ParCopyHelper needs to
4304 // mark the forwarded location of an evacuated object.
4305 // We set _should_mark_objects to true if marking is active, i.e. when we
4306 // need to propagate a mark, or during an initial mark pause, i.e. when we
4307 // need to mark objects immediately reachable by the roots.
4308 if (G1CollectedHeap::heap()->mark_in_progress() ||
4309 G1CollectedHeap::heap()->g1_policy()->during_initial_mark_pause()) {
4310 _should_mark_objects = true;
4311 }
4312 }
4313
4314 G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
4315 : _g1h(g1h),
4316 _refs(g1h->task_queue(queue_num)),
4317 _dcq(&g1h->dirty_card_queue_set()),
4318 _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
4319 _g1_rem(g1h->g1_rem_set()),
4320 _hash_seed(17), _queue_num(queue_num),
4321 _term_attempts(0),
4322 _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived)),
4323 _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)),
4324 _age_table(false),
4325 _strong_roots_time(0), _term_time(0),
4326 _alloc_buffer_waste(0), _undo_waste(0)
4327 {
4328 // we allocate G1YoungSurvRateNumRegions plus one entries, since
4329 // we "sacrifice" entry 0 to keep track of surviving bytes for
4330 // non-young regions (where the age is -1)
4331 // We also add a few elements at the beginning and at the end in
4332 // an attempt to eliminate cache contention
4333 size_t real_length = 1 + _g1h->g1_policy()->young_cset_region_length();
4334 size_t array_length = PADDING_ELEM_NUM +
4335 real_length +
4336 PADDING_ELEM_NUM;
4337 _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
4338 if (_surviving_young_words_base == NULL)
4339 vm_exit_out_of_memory(array_length * sizeof(size_t),
4340 "Not enough space for young surv histo.");
4341 _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
4342 memset(_surviving_young_words, 0, real_length * sizeof(size_t));
4343
4415 assert(_evac_cl != NULL, "not set");
4416 assert(_evac_failure_cl != NULL, "not set");
4417 assert(_partial_scan_cl != NULL, "not set");
4418
4419 StarTask ref;
4420 do {
4421 // Drain the overflow stack first, so other threads can steal.
4422 while (refs()->pop_overflow(ref)) {
4423 deal_with_reference(ref);
4424 }
4425
4426 while (refs()->pop_local(ref)) {
4427 deal_with_reference(ref);
4428 }
4429 } while (!refs()->is_empty());
4430 }
4431
4432 G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
4433 _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
4434 _par_scan_state(par_scan_state),
4435 _during_initial_mark(_g1->g1_policy()->during_initial_mark_pause()),
4436 _mark_in_progress(_g1->mark_in_progress()) { }
4437
4438 template <class T> void G1ParCopyHelper::mark_object(T* p) {
4439 // This is called from do_oop_work for objects that are not
4440 // in the collection set. Objects in the collection set
4441 // are marked after they have been evacuated.
4442
4443 T heap_oop = oopDesc::load_heap_oop(p);
4444 if (!oopDesc::is_null(heap_oop)) {
4445 oop obj = oopDesc::decode_heap_oop(heap_oop);
4446 HeapWord* addr = (HeapWord*)obj;
4447 if (_g1->is_in_g1_reserved(addr)) {
4448 _cm->grayRoot(oop(addr));
4449 }
4450 }
4451 }
4452
4453 oop G1ParCopyHelper::copy_to_survivor_space(oop old, bool should_mark_root,
4454 bool should_mark_copy) {
4455 size_t word_sz = old->size();
4456 HeapRegion* from_region = _g1->heap_region_containing_raw(old);
4457 // +1 to make the -1 indexes valid...
4458 int young_index = from_region->young_index_in_cset()+1;
4459 assert( (from_region->is_young() && young_index > 0) ||
4460 (!from_region->is_young() && young_index == 0), "invariant" );
4461 G1CollectorPolicy* g1p = _g1->g1_policy();
4462 markOop m = old->mark();
4463 int age = m->has_displaced_mark_helper() ? m->displaced_mark_helper()->age()
4464 : m->age();
4465 GCAllocPurpose alloc_purpose = g1p->evacuation_destination(from_region, age,
4466 word_sz);
4467 HeapWord* obj_ptr = _par_scan_state->allocate(alloc_purpose, word_sz);
4468 oop obj = oop(obj_ptr);
4469
4470 if (obj_ptr == NULL) {
4471 // This will either forward-to-self, or detect that someone else has
4472 // installed a forwarding pointer.
4473 OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure();
4474 return _g1->handle_evacuation_failure_par(cl, old, should_mark_root);
4475 }
4476
4477 // We're going to allocate linearly, so might as well prefetch ahead.
4478 Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);
4479
4480 oop forward_ptr = old->forward_to_atomic(obj);
4481 if (forward_ptr == NULL) {
4482 Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz);
4483 if (g1p->track_object_age(alloc_purpose)) {
4484 // We could simply do obj->incr_age(). However, this causes a
4485 // performance issue. obj->incr_age() will first check whether
4486 // the object has a displaced mark by checking its mark word;
4487 // getting the mark word from the new location of the object
4488 // stalls. So, given that we already have the mark word and we
4489 // are about to install it anyway, it's better to increase the
4490 // age on the mark word, when the object does not have a
4491 // displaced mark word. We're not expecting many objects to have
4492 // a displaced marked word, so that case is not optimized
4493 // further (it could be...) and we simply call obj->incr_age().
4494
4497 // otherwise obj looks to be forwarded (the old mark word,
4498 // which contains the forward pointer, was copied)
4499 obj->set_mark(m);
4500 obj->incr_age();
4501 } else {
4502 m = m->incr_age();
4503 obj->set_mark(m);
4504 }
4505 _par_scan_state->age_table()->add(obj, word_sz);
4506 } else {
4507 obj->set_mark(m);
4508 }
4509
4510 // Mark the evacuated object or propagate "next" mark bit
4511 if (should_mark_copy) {
4512 if (!use_local_bitmaps ||
4513 !_par_scan_state->alloc_buffer(alloc_purpose)->mark(obj_ptr)) {
4514 // if we couldn't mark it on the local bitmap (this happens when
4515 // the object was not allocated in the GCLab), we have to bite
4516 // the bullet and do the standard parallel mark
4517 _cm->markAndGrayObjectIfNecessary(obj);
4518 }
4519
4520 if (_g1->isMarkedNext(old)) {
4521 // Unmark the object's old location so that marking
4522 // doesn't think the old object is alive.
4523 _cm->nextMarkBitMap()->parClear((HeapWord*)old);
4524 }
4525 }
4526
4527 size_t* surv_young_words = _par_scan_state->surviving_young_words();
4528 surv_young_words[young_index] += word_sz;
4529
4530 if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
4531 arrayOop(old)->set_length(0);
4532 oop* old_p = set_partial_array_mask(old);
4533 _par_scan_state->push_on_queue(old_p);
4534 } else {
4535 // No point in using the slower heap_region_containing() method,
4536 // given that we know obj is in the heap.
4537 _scanner->set_region(_g1->heap_region_containing_raw(obj));
4538 obj->oop_iterate_backwards(_scanner);
4539 }
4540 } else {
4541 _par_scan_state->undo_allocation(alloc_purpose, obj_ptr, word_sz);
4542 obj = forward_ptr;
4543 }
4590 // during an initial mark pause) to copy_to_survivor_space
4591 // which will pass it on to the evacuation failure handling
4592 // code. The thread that successfully self-forwards a root
4593 // object to itself is responsible for marking the object.
4594 bool should_mark_root = do_mark_object;
4595
4596 // We need to mark the copied object if we're a root scanning
4597 // closure during an initial mark pause (i.e. do_mark_object
4598 // will be true), or the object is already marked and we need
4599 // to propagate the mark to the evacuated copy.
4600 bool should_mark_copy = do_mark_object ||
4601 _during_initial_mark ||
4602 (_mark_in_progress && !_g1->is_obj_ill(obj));
4603
4604 oop copy_oop = copy_to_survivor_space(obj, should_mark_root,
4605 should_mark_copy);
4606 oopDesc::encode_store_heap_oop(p, copy_oop);
4607 }
4608 // When scanning the RS, we only care about objs in CS.
4609 if (barrier == G1BarrierRS) {
4610 _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
4611 }
4612 } else {
4613 // The object is not in collection set. If we're a root scanning
4614 // closure during an initial mark pause (i.e. do_mark_object will
4615 // be true) then attempt to mark the object.
4616 if (do_mark_object) {
4617 mark_object(p);
4618 }
4619 }
4620
4621 if (barrier == G1BarrierEvac && obj != NULL) {
4622 _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
4623 }
4624
4625 if (do_gen_barrier && obj != NULL) {
4626 par_do_barrier(p);
4627 }
4628 }
4629
4630 template void G1ParCopyClosure<false, G1BarrierEvac, false>::do_oop_work(oop* p);
4631 template void G1ParCopyClosure<false, G1BarrierEvac, false>::do_oop_work(narrowOop* p);
4632
4633 template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) {
4634 assert(has_partial_array_mask(p), "invariant");
4635 oop old = clear_partial_array_mask(p);
4636 assert(old->is_objArray(), "must be obj array");
4637 assert(old->is_forwarded(), "must be forwarded");
4638 assert(Universe::heap()->is_in_reserved(old), "must be in heap.");
4639
4640 objArrayOop obj = objArrayOop(old->forwardee());
4641 assert((void*)old != (void*)old->forwardee(), "self forwarding here?");
4642 // Process ParGCArrayScanChunk elements now
5826 assert(index == -1, "invariant");
5827 }
5828
5829 assert( (cur->is_young() && cur->young_index_in_cset() > -1) ||
5830 (!cur->is_young() && cur->young_index_in_cset() == -1),
5831 "invariant" );
5832
5833 if (!cur->evacuation_failed()) {
5834 MemRegion used_mr = cur->used_region();
5835
5836 // And the region is empty.
5837 assert(!used_mr.is_empty(), "Should not have empty regions in a CS.");
5838
5839 // If marking is in progress then clear any objects marked in
5840 // the current region. Note mark_in_progress() returns false,
5841 // even during an initial mark pause, until the set_marking_started()
5842 // call which takes place later in the pause.
5843 if (mark_in_progress()) {
5844 assert(!g1_policy()->during_initial_mark_pause(), "sanity");
5845 _cm->nextMarkBitMap()->clearRange(used_mr);
5846 }
5847
5848 free_region(cur, &pre_used, &local_free_list, false /* par */);
5849 } else {
5850 cur->uninstall_surv_rate_group();
5851 if (cur->is_young()) {
5852 cur->set_young_index_in_cset(-1);
5853 }
5854 cur->set_not_young();
5855 cur->set_evacuation_failed(false);
5856 // The region is now considered to be old.
5857 _old_set.add(cur);
5858 }
5859 cur = next;
5860 }
5861
5862 policy->record_max_rs_lengths(rs_lengths);
5863 policy->cset_regions_freed();
5864
5865 double end_sec = os::elapsedTime();
5866 double elapsed_ms = (end_sec - start_sec) * 1000.0;
5867
|
4037 // would point into middle of the filler object.
4038 // The current approach is to not coalesce and leave the BOT contents intact.
4039 // </original comment>
4040 //
4041 // We now reset the BOT when we start the object iteration over the
4042 // region and refine its entries for every object we come across. So
4043 // the above comment is not really relevant and we should be able
4044 // to coalesce dead objects if we want to.
4045 void do_object(oop obj) {
4046 HeapWord* obj_addr = (HeapWord*) obj;
4047 assert(_hr->is_in(obj_addr), "sanity");
4048 size_t obj_size = obj->size();
4049 _hr->update_bot_for_object(obj_addr, obj_size);
4050 if (obj->is_forwarded() && obj->forwardee() == obj) {
4051 // The object failed to move.
4052 assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs.");
4053 _cm->markPrev(obj);
4054 assert(_cm->isPrevMarked(obj), "Should be marked!");
4055 _prev_marked_bytes += (obj_size * HeapWordSize);
4056 if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) {
4057 _cm->markAndGrayObjectIfNecessary(obj, 0 /* worker_i */);
4058 }
4059 obj->set_mark(markOopDesc::prototype());
4060 // While we were processing RSet buffers during the
4061 // collection, we actually didn't scan any cards on the
4062 // collection set, since we didn't want to update remebered
4063 // sets with entries that point into the collection set, given
4064 // that live objects fromthe collection set are about to move
4065 // and such entries will be stale very soon. This change also
4066 // dealt with a reliability issue which involved scanning a
4067 // card in the collection set and coming across an array that
4068 // was being chunked and looking malformed. The problem is
4069 // that, if evacuation fails, we might have remembered set
4070 // entries missing given that we skipped cards on the
4071 // collection set. So, we'll recreate such entries now.
4072 obj->oop_iterate(_cl);
4073 assert(_cm->isPrevMarked(obj), "Should be marked!");
4074 } else {
4075 // The object has been either evacuated or is dead. Fill it with a
4076 // dummy object.
4077 MemRegion mr((HeapWord*)obj, obj_size);
4156 }
4157 }
4158
4159 void G1CollectedHeap::push_on_evac_failure_scan_stack(oop obj) {
4160 _evac_failure_scan_stack->push(obj);
4161 }
4162
4163 void G1CollectedHeap::drain_evac_failure_scan_stack() {
4164 assert(_evac_failure_scan_stack != NULL, "precondition");
4165
4166 while (_evac_failure_scan_stack->length() > 0) {
4167 oop obj = _evac_failure_scan_stack->pop();
4168 _evac_failure_closure->set_region(heap_region_containing(obj));
4169 obj->oop_iterate_backwards(_evac_failure_closure);
4170 }
4171 }
4172
4173 oop
4174 G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl,
4175 oop old,
4176 bool should_mark_root,
4177 int worker_i) {
4178 assert(obj_in_cs(old),
4179 err_msg("obj: "PTR_FORMAT" should still be in the CSet",
4180 (HeapWord*) old));
4181 markOop m = old->mark();
4182 oop forward_ptr = old->forward_to_atomic(old);
4183 if (forward_ptr == NULL) {
4184 // Forward-to-self succeeded.
4185
4186 // should_mark_root will be true when this routine is called
4187 // from a root scanning closure during an initial mark pause.
4188 // In this case the thread that succeeds in self-forwarding the
4189 // object is also responsible for marking the object.
4190 if (should_mark_root) {
4191 assert(!oopDesc::is_null(old), "shouldn't be");
4192 _cm->grayRoot(old, worker_i);
4193 }
4194
4195 if (_evac_failure_closure != cl) {
4196 MutexLockerEx x(EvacFailureStack_lock, Mutex::_no_safepoint_check_flag);
4197 assert(!_drain_in_progress,
4198 "Should only be true while someone holds the lock.");
4199 // Set the global evac-failure closure to the current thread's.
4200 assert(_evac_failure_closure == NULL, "Or locking has failed.");
4201 set_evac_failure_closure(cl);
4202 // Now do the common part.
4203 handle_evacuation_failure_common(old, m);
4204 // Reset to NULL.
4205 set_evac_failure_closure(NULL);
4206 } else {
4207 // The lock is already held, and this is recursive.
4208 assert(_drain_in_progress, "This should only be the recursive case.");
4209 handle_evacuation_failure_common(old, m);
4210 }
4211 return old;
4212 } else {
4278 } else {
4279 // Let's try to allocate in the survivors in case we can fit the
4280 // object there.
4281 return survivor_attempt_allocation(word_size);
4282 }
4283 }
4284
4285 ShouldNotReachHere();
4286 // Trying to keep some compilers happy.
4287 return NULL;
4288 }
4289
4290 #ifndef PRODUCT
4291 bool GCLabBitMapClosure::do_bit(size_t offset) {
4292 HeapWord* addr = _bitmap->offsetToHeapWord(offset);
4293 guarantee(_cm->isMarked(oop(addr)), "it should be!");
4294 return true;
4295 }
4296 #endif // PRODUCT
4297
4298 void GCLabBitMap::retire(int worker_i) {
4299 guarantee(use_local_bitmaps, "invariant");
4300 assert(fields_well_formed(), "invariant");
4301
4302 if (_start_word != NULL) {
4303 CMBitMap* mark_bitmap = _cm->nextMarkBitMap();
4304
4305 // this means that the bitmap was set up for the GCLab
4306 assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
4307
4308 mark_bitmap->mostly_disjoint_range_union(this,
4309 0, // always start from the start of the bitmap
4310 _start_word,
4311 gclab_real_word_size());
4312
4313 // Note: Even though that not all objects copied into the LAB will
4314 // have their bit set in the LAB bitmap (the LAB bitmap is used to
4315 // propagate marks), we can just add the entire lab and its bitmap
4316 // to the count of the marked data. It's OK (but inaccurate) to
4317 // count a dead object but we can't miss counting a live object.
4318 MemRegion lab_region(_real_start_word, _real_end_word);
4319 _cm->count_region(lab_region, worker_i);
4320 _cm->grayRegionIfNecessary(lab_region);
4321
4322 #ifndef PRODUCT
4323 if (use_local_bitmaps && verify_local_bitmaps) {
4324 verify();
4325 }
4326 #endif // PRODUCT
4327 } else {
4328 assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
4329 }
4330 }
4331
4332 G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size, int worker_i) :
4333 ParGCAllocBuffer(gclab_word_size),
4334 _should_mark_objects(false),
4335 _bitmap(G1CollectedHeap::heap()->reserved_region().start(), gclab_word_size),
4336 _worker_i(worker_i),
4337 _retired(false)
4338 {
4339 //_should_mark_objects is set to true when G1ParCopyHelper needs to
4340 // mark the forwarded location of an evacuated object.
4341 // We set _should_mark_objects to true if marking is active, i.e. when we
4342 // need to propagate a mark, or during an initial mark pause, i.e. when we
4343 // need to mark objects immediately reachable by the roots.
4344 if (G1CollectedHeap::heap()->mark_in_progress() ||
4345 G1CollectedHeap::heap()->g1_policy()->during_initial_mark_pause()) {
4346 _should_mark_objects = true;
4347 }
4348 }
4349
4350 G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
4351 : _g1h(g1h),
4352 _refs(g1h->task_queue(queue_num)),
4353 _dcq(&g1h->dirty_card_queue_set()),
4354 _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
4355 _g1_rem(g1h->g1_rem_set()),
4356 _hash_seed(17), _queue_num(queue_num),
4357 _term_attempts(0),
4358 _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived), queue_num),
4359 _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured), queue_num),
4360 _age_table(false),
4361 _strong_roots_time(0), _term_time(0),
4362 _alloc_buffer_waste(0), _undo_waste(0)
4363 {
4364 // we allocate G1YoungSurvRateNumRegions plus one entries, since
4365 // we "sacrifice" entry 0 to keep track of surviving bytes for
4366 // non-young regions (where the age is -1)
4367 // We also add a few elements at the beginning and at the end in
4368 // an attempt to eliminate cache contention
4369 size_t real_length = 1 + _g1h->g1_policy()->young_cset_region_length();
4370 size_t array_length = PADDING_ELEM_NUM +
4371 real_length +
4372 PADDING_ELEM_NUM;
4373 _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
4374 if (_surviving_young_words_base == NULL)
4375 vm_exit_out_of_memory(array_length * sizeof(size_t),
4376 "Not enough space for young surv histo.");
4377 _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
4378 memset(_surviving_young_words, 0, real_length * sizeof(size_t));
4379
4451 assert(_evac_cl != NULL, "not set");
4452 assert(_evac_failure_cl != NULL, "not set");
4453 assert(_partial_scan_cl != NULL, "not set");
4454
4455 StarTask ref;
4456 do {
4457 // Drain the overflow stack first, so other threads can steal.
4458 while (refs()->pop_overflow(ref)) {
4459 deal_with_reference(ref);
4460 }
4461
4462 while (refs()->pop_local(ref)) {
4463 deal_with_reference(ref);
4464 }
4465 } while (!refs()->is_empty());
4466 }
4467
4468 G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
4469 _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
4470 _par_scan_state(par_scan_state),
4471 _worker_i(par_scan_state->queue_num()),
4472 _during_initial_mark(_g1->g1_policy()->during_initial_mark_pause()),
4473 _mark_in_progress(_g1->mark_in_progress()) { }
4474
4475 template <class T> void G1ParCopyHelper::mark_object(T* p) {
4476 // This is called from do_oop_work for objects that are not
4477 // in the collection set. Objects in the collection set
4478 // are marked after they have been evacuated.
4479
4480 T heap_oop = oopDesc::load_heap_oop(p);
4481 if (!oopDesc::is_null(heap_oop)) {
4482 oop obj = oopDesc::decode_heap_oop(heap_oop);
4483 HeapWord* addr = (HeapWord*)obj;
4484 if (_g1->is_in_g1_reserved(addr)) {
4485 _cm->grayRoot(oop(addr), _worker_i);
4486 }
4487 }
4488 }
4489
4490 oop G1ParCopyHelper::copy_to_survivor_space(oop old, bool should_mark_root,
4491 bool should_mark_copy) {
4492 size_t word_sz = old->size();
4493 HeapRegion* from_region = _g1->heap_region_containing_raw(old);
4494 // +1 to make the -1 indexes valid...
4495 int young_index = from_region->young_index_in_cset()+1;
4496 assert( (from_region->is_young() && young_index > 0) ||
4497 (!from_region->is_young() && young_index == 0), "invariant" );
4498 G1CollectorPolicy* g1p = _g1->g1_policy();
4499 markOop m = old->mark();
4500 int age = m->has_displaced_mark_helper() ? m->displaced_mark_helper()->age()
4501 : m->age();
4502 GCAllocPurpose alloc_purpose = g1p->evacuation_destination(from_region, age,
4503 word_sz);
4504 HeapWord* obj_ptr = _par_scan_state->allocate(alloc_purpose, word_sz);
4505 oop obj = oop(obj_ptr);
4506
4507 if (obj_ptr == NULL) {
4508 // This will either forward-to-self, or detect that someone else has
4509 // installed a forwarding pointer.
4510 OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure();
4511 return _g1->handle_evacuation_failure_par(cl, old, should_mark_root, _worker_i);
4512 }
4513
4514 // We're going to allocate linearly, so might as well prefetch ahead.
4515 Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);
4516
4517 oop forward_ptr = old->forward_to_atomic(obj);
4518 if (forward_ptr == NULL) {
4519 Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz);
4520 if (g1p->track_object_age(alloc_purpose)) {
4521 // We could simply do obj->incr_age(). However, this causes a
4522 // performance issue. obj->incr_age() will first check whether
4523 // the object has a displaced mark by checking its mark word;
4524 // getting the mark word from the new location of the object
4525 // stalls. So, given that we already have the mark word and we
4526 // are about to install it anyway, it's better to increase the
4527 // age on the mark word, when the object does not have a
4528 // displaced mark word. We're not expecting many objects to have
4529 // a displaced marked word, so that case is not optimized
4530 // further (it could be...) and we simply call obj->incr_age().
4531
4534 // otherwise obj looks to be forwarded (the old mark word,
4535 // which contains the forward pointer, was copied)
4536 obj->set_mark(m);
4537 obj->incr_age();
4538 } else {
4539 m = m->incr_age();
4540 obj->set_mark(m);
4541 }
4542 _par_scan_state->age_table()->add(obj, word_sz);
4543 } else {
4544 obj->set_mark(m);
4545 }
4546
4547 // Mark the evacuated object or propagate "next" mark bit
4548 if (should_mark_copy) {
4549 if (!use_local_bitmaps ||
4550 !_par_scan_state->alloc_buffer(alloc_purpose)->mark(obj_ptr)) {
4551 // if we couldn't mark it on the local bitmap (this happens when
4552 // the object was not allocated in the GCLab), we have to bite
4553 // the bullet and do the standard parallel mark
4554 _cm->markAndGrayObjectIfNecessary(obj, _worker_i);
4555 }
4556
4557 if (_g1->isMarkedNext(old)) {
4558 // Unmark the object's old location so that marking
4559 // doesn't think the old object is alive.
4560 _cm->nextMarkBitMap()->parClear((HeapWord*)old);
4561
4562 // We could clear the count data for the old object here but
4563 // currently we do not. Why don't we do this? The thread/task
4564 // that marks a newly copied object is likely _not_ the thread/task
4565 // that originally marked the old object. So, to clear the count
4566 // data for the old object, we would have to scan the count
4567 // data for all of the tasks (and clear the data for the old object
4568 // in parallel with other threads adding to the count data). Even
4569 // then we could clear a bit incorrectly (e.g. if the old object
4570 // does not start or end on a card boundary). It's more important
4571 // that we don't have missed bits that should've been set than
4572 // having extra bits set.
4573 //
4574 // As a result the accumulated count data could be a superset
4575 // of the data that is/would have been calculated by walking
4576 // the marking bitmap.
4577 }
4578 }
4579
4580 size_t* surv_young_words = _par_scan_state->surviving_young_words();
4581 surv_young_words[young_index] += word_sz;
4582
4583 if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
4584 arrayOop(old)->set_length(0);
4585 oop* old_p = set_partial_array_mask(old);
4586 _par_scan_state->push_on_queue(old_p);
4587 } else {
4588 // No point in using the slower heap_region_containing() method,
4589 // given that we know obj is in the heap.
4590 _scanner->set_region(_g1->heap_region_containing_raw(obj));
4591 obj->oop_iterate_backwards(_scanner);
4592 }
4593 } else {
4594 _par_scan_state->undo_allocation(alloc_purpose, obj_ptr, word_sz);
4595 obj = forward_ptr;
4596 }
4643 // during an initial mark pause) to copy_to_survivor_space
4644 // which will pass it on to the evacuation failure handling
4645 // code. The thread that successfully self-forwards a root
4646 // object to itself is responsible for marking the object.
4647 bool should_mark_root = do_mark_object;
4648
4649 // We need to mark the copied object if we're a root scanning
4650 // closure during an initial mark pause (i.e. do_mark_object
4651 // will be true), or the object is already marked and we need
4652 // to propagate the mark to the evacuated copy.
4653 bool should_mark_copy = do_mark_object ||
4654 _during_initial_mark ||
4655 (_mark_in_progress && !_g1->is_obj_ill(obj));
4656
4657 oop copy_oop = copy_to_survivor_space(obj, should_mark_root,
4658 should_mark_copy);
4659 oopDesc::encode_store_heap_oop(p, copy_oop);
4660 }
4661 // When scanning the RS, we only care about objs in CS.
4662 if (barrier == G1BarrierRS) {
4663 assert(_worker_i == _par_scan_state->queue_num(), "sanity");
4664 _par_scan_state->update_rs(_from, p, _worker_i);
4665 }
4666 } else {
4667 // The object is not in collection set. If we're a root scanning
4668 // closure during an initial mark pause (i.e. do_mark_object will
4669 // be true) then attempt to mark the object.
4670 if (do_mark_object) {
4671 mark_object(p);
4672 }
4673 }
4674
4675 if (barrier == G1BarrierEvac && obj != NULL) {
4676 assert(_worker_i == _par_scan_state->queue_num(), "sanity");
4677 _par_scan_state->update_rs(_from, p, _worker_i);
4678 }
4679
4680 if (do_gen_barrier && obj != NULL) {
4681 par_do_barrier(p);
4682 }
4683 }
4684
4685 template void G1ParCopyClosure<false, G1BarrierEvac, false>::do_oop_work(oop* p);
4686 template void G1ParCopyClosure<false, G1BarrierEvac, false>::do_oop_work(narrowOop* p);
4687
4688 template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) {
4689 assert(has_partial_array_mask(p), "invariant");
4690 oop old = clear_partial_array_mask(p);
4691 assert(old->is_objArray(), "must be obj array");
4692 assert(old->is_forwarded(), "must be forwarded");
4693 assert(Universe::heap()->is_in_reserved(old), "must be in heap.");
4694
4695 objArrayOop obj = objArrayOop(old->forwardee());
4696 assert((void*)old != (void*)old->forwardee(), "self forwarding here?");
4697 // Process ParGCArrayScanChunk elements now
5881 assert(index == -1, "invariant");
5882 }
5883
5884 assert( (cur->is_young() && cur->young_index_in_cset() > -1) ||
5885 (!cur->is_young() && cur->young_index_in_cset() == -1),
5886 "invariant" );
5887
5888 if (!cur->evacuation_failed()) {
5889 MemRegion used_mr = cur->used_region();
5890
5891 // And the region is empty.
5892 assert(!used_mr.is_empty(), "Should not have empty regions in a CS.");
5893
5894 // If marking is in progress then clear any objects marked in
5895 // the current region. Note mark_in_progress() returns false,
5896 // even during an initial mark pause, until the set_marking_started()
5897 // call which takes place later in the pause.
5898 if (mark_in_progress()) {
5899 assert(!g1_policy()->during_initial_mark_pause(), "sanity");
5900 _cm->nextMarkBitMap()->clearRange(used_mr);
5901 // Need to remove values from the count info
5902 _cm->clear_count_data_for_heap_region(cur);
5903 }
5904 free_region(cur, &pre_used, &local_free_list, false /* par */);
5905 } else {
5906 cur->uninstall_surv_rate_group();
5907 if (cur->is_young()) {
5908 cur->set_young_index_in_cset(-1);
5909 }
5910 cur->set_not_young();
5911 cur->set_evacuation_failed(false);
5912 // The region is now considered to be old.
5913 _old_set.add(cur);
5914 }
5915 cur = next;
5916 }
5917
5918 policy->record_max_rs_lengths(rs_lengths);
5919 policy->cset_regions_freed();
5920
5921 double end_sec = os::elapsedTime();
5922 double elapsed_ms = (end_sec - start_sec) * 1000.0;
5923
|