2961
2962 // read it again
2963 finger = _finger;
2964 }
2965 }
2966
2967 return NULL;
2968 }
2969
2970 #ifndef PRODUCT
2971 enum VerifyNoCSetOopsPhase {
2972 VerifyNoCSetOopsStack,
2973 VerifyNoCSetOopsQueues,
2974 VerifyNoCSetOopsSATBCompleted,
2975 VerifyNoCSetOopsSATBThread
2976 };
2977
2978 class VerifyNoCSetOopsClosure : public OopClosure, public ObjectClosure {
2979 private:
2980 G1CollectedHeap* _g1h;
2981 VerifyNoCSetOopsPhase _phase;
2982 int _info;
2983
2984 const char* phase_str() {
2985 switch (_phase) {
2986 case VerifyNoCSetOopsStack: return "Stack";
2987 case VerifyNoCSetOopsQueues: return "Queue";
2988 case VerifyNoCSetOopsSATBCompleted: return "Completed SATB Buffers";
2989 case VerifyNoCSetOopsSATBThread: return "Thread SATB Buffers";
2990 default: ShouldNotReachHere();
2991 }
2992 return NULL;
2993 }
2994
2995 void do_object_work(oop obj) {
2996 guarantee(!_g1h->obj_in_cs(obj),
2997 err_msg("obj: "PTR_FORMAT" in CSet, phase: %s, info: %d",
2998 p2i((void*) obj), phase_str(), _info));
2999 }
3000
3001 public:
3002 VerifyNoCSetOopsClosure() : _g1h(G1CollectedHeap::heap()) { }
3003
3004 void set_phase(VerifyNoCSetOopsPhase phase, int info = -1) {
3005 _phase = phase;
3006 _info = info;
3007 }
3008
3009 virtual void do_oop(oop* p) {
3010 oop obj = oopDesc::load_decode_heap_oop(p);
3011 do_object_work(obj);
3012 }
3013
3014 virtual void do_oop(narrowOop* p) {
3015 // We should not come across narrow oops while scanning marking
3016 // stacks and SATB buffers.
3017 ShouldNotReachHere();
3018 }
3019
3020 virtual void do_object(oop obj) {
3021 do_object_work(obj);
3022 }
3023 };
3024
3025 void ConcurrentMark::verify_no_cset_oops(bool verify_stacks,
3026 bool verify_enqueued_buffers,
3027 bool verify_thread_buffers,
3028 bool verify_fingers) {
3029 assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint");
3030 if (!G1CollectedHeap::heap()->mark_in_progress()) {
3031 return;
3032 }
3033
3034 VerifyNoCSetOopsClosure cl;
3035
3036 if (verify_stacks) {
3037 // Verify entries on the global mark stack
3038 cl.set_phase(VerifyNoCSetOopsStack);
3039 _markStack.oops_do(&cl);
3040
3041 // Verify entries on the task queues
3042 for (uint i = 0; i < _max_worker_id; i += 1) {
3043 cl.set_phase(VerifyNoCSetOopsQueues, i);
3044 CMTaskQueue* queue = _task_queues->queue(i);
3045 queue->oops_do(&cl);
3046 }
3047 }
3048
3049 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
3050
3051 // Verify entries on the enqueued SATB buffers
3052 if (verify_enqueued_buffers) {
3053 cl.set_phase(VerifyNoCSetOopsSATBCompleted);
3054 satb_qs.iterate_completed_buffers_read_only(&cl);
3389 return true;
3390 } else {
3391 return false;
3392 }
3393 }
3394
3395 #ifndef PRODUCT
3396 // for debugging purposes
3397 void ConcurrentMark::print_finger() {
3398 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT,
3399 p2i(_heap_start), p2i(_heap_end), p2i(_finger));
3400 for (uint i = 0; i < _max_worker_id; ++i) {
3401 gclog_or_tty->print(" %u: " PTR_FORMAT, i, p2i(_tasks[i]->finger()));
3402 }
3403 gclog_or_tty->cr();
3404 }
3405 #endif
3406
3407 void CMTask::scan_object(oop obj) {
3408 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant");
3409
3410 if (_cm->verbose_high()) {
3411 gclog_or_tty->print_cr("[%u] we're scanning object "PTR_FORMAT,
3412 _worker_id, p2i((void*) obj));
3413 }
3414
3415 size_t obj_size = obj->size();
3416 _words_scanned += obj_size;
3417
3418 obj->oop_iterate(_cm_oop_closure);
3419 statsOnly( ++_objs_scanned );
3420 check_limits();
3421 }
3422
3423 // Closure for iteration over bitmaps
3424 class CMBitMapClosure : public BitMapClosure {
3425 private:
3426 // the bitmap that is being iterated over
3427 CMBitMap* _nextMarkBitMap;
3428 ConcurrentMark* _cm;
3801 } else {
3802 target_size = 0;
3803 }
3804
3805 if (_task_queue->size() > target_size) {
3806 if (_cm->verbose_high()) {
3807 gclog_or_tty->print_cr("[%u] draining local queue, target size = " SIZE_FORMAT,
3808 _worker_id, target_size);
3809 }
3810
3811 oop obj;
3812 bool ret = _task_queue->pop_local(obj);
3813 while (ret) {
3814 statsOnly( ++_local_pops );
3815
3816 if (_cm->verbose_high()) {
3817 gclog_or_tty->print_cr("[%u] popped "PTR_FORMAT, _worker_id,
3818 p2i((void*) obj));
3819 }
3820
3821 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" );
3822
3823 if (is_stale_humongous_queue_entry(obj)) {
3824 statsOnly( ++stale_humongous_queue_entries );
3825 } else {
3826 assert(!_g1h->is_on_master_free_list(
3827 _g1h->heap_region_containing(obj)), "invariant");
3828 scan_object(obj);
3829 }
3830
3831 if (_task_queue->size() <= target_size || has_aborted()) {
3832 ret = false;
3833 } else {
3834 ret = _task_queue->pop_local(obj);
3835 }
3836 }
3837
3838 if (_cm->verbose_high()) {
3839 gclog_or_tty->print_cr("[%u] drained local queue, size = %u",
3840 _worker_id, _task_queue->size());
3841 }
3842 }
3843 }
3844
3845 void CMTask::drain_global_stack(bool partially) {
3846 if (has_aborted()) return;
3847
3848 // We have a policy to drain the local queue before we attempt to
3849 // drain the global stack.
4313 // We cannot check whether the global stack is empty, since other
4314 // tasks might be pushing objects to it concurrently.
4315 assert(_cm->out_of_regions() && _task_queue->size() == 0,
4316 "only way to reach here");
4317
4318 if (_cm->verbose_low()) {
4319 gclog_or_tty->print_cr("[%u] starting to steal", _worker_id);
4320 }
4321
4322 while (!has_aborted()) {
4323 oop obj;
4324 statsOnly( ++_steal_attempts );
4325
4326 if (_cm->try_stealing(_worker_id, &_hash_seed, obj)) {
4327 if (_cm->verbose_medium()) {
4328 gclog_or_tty->print_cr("[%u] stolen "PTR_FORMAT" successfully",
4329 _worker_id, p2i((void*) obj));
4330 }
4331
4332 statsOnly( ++_steals );
4333
4334 if (is_stale_humongous_queue_entry(obj)) {
4335 statsOnly( ++stale_humongous_queue_entries );
4336 } else {
4337 assert(_nextMarkBitMap->isMarked((HeapWord*) obj),
4338 "any stolen object should be marked");
4339 scan_object(obj);
4340 }
4341
4342 // And since we're towards the end, let's totally drain the
4343 // local queue and global stack.
4344 drain_local_queue(false);
4345 drain_global_stack(false);
4346 } else {
4347 break;
4348 }
4349 }
4350 }
4351
4352 // If we are about to wrap up and go into termination, check if we
4353 // should raise the overflow flag.
4354 if (do_termination && !has_aborted()) {
4355 if (_cm->force_overflow()->should_force()) {
4356 _cm->set_has_overflown();
4357 regular_clock_call();
4358 }
4359 }
4360
|
2961
2962 // read it again
2963 finger = _finger;
2964 }
2965 }
2966
2967 return NULL;
2968 }
2969
2970 #ifndef PRODUCT
2971 enum VerifyNoCSetOopsPhase {
2972 VerifyNoCSetOopsStack,
2973 VerifyNoCSetOopsQueues,
2974 VerifyNoCSetOopsSATBCompleted,
2975 VerifyNoCSetOopsSATBThread
2976 };
2977
2978 class VerifyNoCSetOopsClosure : public OopClosure, public ObjectClosure {
2979 private:
2980 G1CollectedHeap* _g1h;
2981 ConcurrentMark* _cm;
2982 VerifyNoCSetOopsPhase _phase;
2983 int _info;
2984
2985 const char* phase_str() {
2986 switch (_phase) {
2987 case VerifyNoCSetOopsStack: return "Stack";
2988 case VerifyNoCSetOopsQueues: return "Queue";
2989 case VerifyNoCSetOopsSATBCompleted: return "Completed SATB Buffers";
2990 case VerifyNoCSetOopsSATBThread: return "Thread SATB Buffers";
2991 default: ShouldNotReachHere();
2992 }
2993 return NULL;
2994 }
2995
2996 void do_object_work(oop obj) {
2997 switch (_phase) {
2998 case VerifyNoCSetOopsStack:
2999 case VerifyNoCSetOopsQueues:
3000 // Ignore reclaimed humongous object entries in mark stack and
3001 // thread queues.
3002 if (_cm->is_stale_humongous_marked_entry(obj)) break;
3003 default:
3004 guarantee(!_g1h->obj_in_cs(obj),
3005 err_msg("obj: "PTR_FORMAT" in CSet, phase: %s, info: %d",
3006 p2i((void*) obj), phase_str(), _info));
3007 }
3008 }
3009
3010 public:
3011 VerifyNoCSetOopsClosure(G1CollectedHeap* g1h, ConcurrentMark* cm)
3012 : _g1h(g1h), _cm(cm) { }
3013
3014 void set_phase(VerifyNoCSetOopsPhase phase, int info = -1) {
3015 _phase = phase;
3016 _info = info;
3017 }
3018
3019 virtual void do_oop(oop* p) {
3020 oop obj = oopDesc::load_decode_heap_oop(p);
3021 do_object_work(obj);
3022 }
3023
3024 virtual void do_oop(narrowOop* p) {
3025 // We should not come across narrow oops while scanning marking
3026 // stacks and SATB buffers.
3027 ShouldNotReachHere();
3028 }
3029
3030 virtual void do_object(oop obj) {
3031 do_object_work(obj);
3032 }
3033 };
3034
3035 void ConcurrentMark::verify_no_cset_oops(bool verify_stacks,
3036 bool verify_enqueued_buffers,
3037 bool verify_thread_buffers,
3038 bool verify_fingers) {
3039 assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint");
3040 if (!G1CollectedHeap::heap()->mark_in_progress()) {
3041 return;
3042 }
3043
3044 VerifyNoCSetOopsClosure cl(_g1h, this);
3045
3046 if (verify_stacks) {
3047 // Verify entries on the global mark stack
3048 cl.set_phase(VerifyNoCSetOopsStack);
3049 _markStack.oops_do(&cl);
3050
3051 // Verify entries on the task queues
3052 for (uint i = 0; i < _max_worker_id; i += 1) {
3053 cl.set_phase(VerifyNoCSetOopsQueues, i);
3054 CMTaskQueue* queue = _task_queues->queue(i);
3055 queue->oops_do(&cl);
3056 }
3057 }
3058
3059 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
3060
3061 // Verify entries on the enqueued SATB buffers
3062 if (verify_enqueued_buffers) {
3063 cl.set_phase(VerifyNoCSetOopsSATBCompleted);
3064 satb_qs.iterate_completed_buffers_read_only(&cl);
3399 return true;
3400 } else {
3401 return false;
3402 }
3403 }
3404
3405 #ifndef PRODUCT
3406 // for debugging purposes
3407 void ConcurrentMark::print_finger() {
3408 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT,
3409 p2i(_heap_start), p2i(_heap_end), p2i(_finger));
3410 for (uint i = 0; i < _max_worker_id; ++i) {
3411 gclog_or_tty->print(" %u: " PTR_FORMAT, i, p2i(_tasks[i]->finger()));
3412 }
3413 gclog_or_tty->cr();
3414 }
3415 #endif
3416
3417 void CMTask::scan_object(oop obj) {
3418 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant");
3419 assert(!_g1h->is_on_master_free_list(_g1h->heap_region_containing(obj)), "invariant");
3420
3421 if (_cm->verbose_high()) {
3422 gclog_or_tty->print_cr("[%u] we're scanning object "PTR_FORMAT,
3423 _worker_id, p2i((void*) obj));
3424 }
3425
3426 size_t obj_size = obj->size();
3427 _words_scanned += obj_size;
3428
3429 obj->oop_iterate(_cm_oop_closure);
3430 statsOnly( ++_objs_scanned );
3431 check_limits();
3432 }
3433
3434 // Closure for iteration over bitmaps
3435 class CMBitMapClosure : public BitMapClosure {
3436 private:
3437 // the bitmap that is being iterated over
3438 CMBitMap* _nextMarkBitMap;
3439 ConcurrentMark* _cm;
3812 } else {
3813 target_size = 0;
3814 }
3815
3816 if (_task_queue->size() > target_size) {
3817 if (_cm->verbose_high()) {
3818 gclog_or_tty->print_cr("[%u] draining local queue, target size = " SIZE_FORMAT,
3819 _worker_id, target_size);
3820 }
3821
3822 oop obj;
3823 bool ret = _task_queue->pop_local(obj);
3824 while (ret) {
3825 statsOnly( ++_local_pops );
3826
3827 if (_cm->verbose_high()) {
3828 gclog_or_tty->print_cr("[%u] popped "PTR_FORMAT, _worker_id,
3829 p2i((void*) obj));
3830 }
3831
3832 process_queue_entry(obj);
3833
3834 if (_task_queue->size() <= target_size || has_aborted()) {
3835 ret = false;
3836 } else {
3837 ret = _task_queue->pop_local(obj);
3838 }
3839 }
3840
3841 if (_cm->verbose_high()) {
3842 gclog_or_tty->print_cr("[%u] drained local queue, size = %u",
3843 _worker_id, _task_queue->size());
3844 }
3845 }
3846 }
3847
3848 void CMTask::drain_global_stack(bool partially) {
3849 if (has_aborted()) return;
3850
3851 // We have a policy to drain the local queue before we attempt to
3852 // drain the global stack.
4316 // We cannot check whether the global stack is empty, since other
4317 // tasks might be pushing objects to it concurrently.
4318 assert(_cm->out_of_regions() && _task_queue->size() == 0,
4319 "only way to reach here");
4320
4321 if (_cm->verbose_low()) {
4322 gclog_or_tty->print_cr("[%u] starting to steal", _worker_id);
4323 }
4324
4325 while (!has_aborted()) {
4326 oop obj;
4327 statsOnly( ++_steal_attempts );
4328
4329 if (_cm->try_stealing(_worker_id, &_hash_seed, obj)) {
4330 if (_cm->verbose_medium()) {
4331 gclog_or_tty->print_cr("[%u] stolen "PTR_FORMAT" successfully",
4332 _worker_id, p2i((void*) obj));
4333 }
4334
4335 statsOnly( ++_steals );
4336 process_queue_entry(obj);
4337
4338 // And since we're towards the end, let's totally drain the
4339 // local queue and global stack.
4340 drain_local_queue(false);
4341 drain_global_stack(false);
4342 } else {
4343 break;
4344 }
4345 }
4346 }
4347
4348 // If we are about to wrap up and go into termination, check if we
4349 // should raise the overflow flag.
4350 if (do_termination && !has_aborted()) {
4351 if (_cm->force_overflow()->should_force()) {
4352 _cm->set_has_overflown();
4353 regular_clock_call();
4354 }
4355 }
4356
|