43 #include "gc/shared/gcTimer.hpp"
44 #include "gc/shared/gcTrace.hpp"
45 #include "gc/shared/gcTraceTime.inline.hpp"
46 #include "gc/shared/genOopClosures.inline.hpp"
47 #include "gc/shared/referencePolicy.hpp"
48 #include "gc/shared/strongRootsScope.hpp"
49 #include "gc/shared/taskqueue.inline.hpp"
50 #include "gc/shared/vmGCOperations.hpp"
51 #include "logging/log.hpp"
52 #include "memory/allocation.hpp"
53 #include "memory/resourceArea.hpp"
54 #include "oops/oop.inline.hpp"
55 #include "runtime/atomic.hpp"
56 #include "runtime/handles.inline.hpp"
57 #include "runtime/java.hpp"
58 #include "runtime/prefetch.inline.hpp"
59 #include "services/memTracker.hpp"
60 #include "utilities/align.hpp"
61 #include "utilities/growableArray.hpp"
62
63 // Concurrent marking bit map wrapper
64
65 G1CMBitMapRO::G1CMBitMapRO(int shifter) :
66 _bm(),
67 _shifter(shifter) {
68 _bmStartWord = 0;
69 _bmWordSize = 0;
70 }
71
72 HeapWord* G1CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
73 const HeapWord* limit) const {
74 // First we must round addr *up* to a possible object boundary.
75 addr = align_up(addr, HeapWordSize << _shifter);
76 size_t addrOffset = heapWordToOffset(addr);
77 assert(limit != NULL, "limit must not be NULL");
78 size_t limitOffset = heapWordToOffset(limit);
79 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
80 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
81 assert(nextAddr >= addr, "get_next_one postcondition");
82 assert(nextAddr == limit || isMarked(nextAddr),
83 "get_next_one postcondition");
84 return nextAddr;
85 }
86
87 #ifndef PRODUCT
88 bool G1CMBitMapRO::covers(MemRegion heap_rs) const {
89 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
90 assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize,
91 "size inconsistency");
92 return _bmStartWord == (HeapWord*)(heap_rs.start()) &&
93 _bmWordSize == heap_rs.word_size();
94 }
95 #endif
96
97 void G1CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const {
98 _bm.print_on_error(st, prefix);
99 }
100
101 size_t G1CMBitMap::compute_size(size_t heap_size) {
102 return ReservedSpace::allocation_align_size_up(heap_size / mark_distance());
103 }
104
105 size_t G1CMBitMap::mark_distance() {
106 return MinObjAlignmentInBytes * BitsPerByte;
107 }
108
109 void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) {
110 _bmStartWord = heap.start();
111 _bmWordSize = heap.word_size();
112
113 _bm = BitMapView((BitMap::bm_word_t*) storage->reserved().start(), _bmWordSize >> _shifter);
114
115 storage->set_mapping_changed_listener(&_listener);
116 }
117
118 void G1CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) {
119 if (zero_filled) {
120 return;
121 }
122 // We need to clear the bitmap on commit, removing any existing information.
123 MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords);
124 _bm->clear_range(mr);
125 }
126
127 void G1CMBitMap::clear_range(MemRegion mr) {
128 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
129 assert(!mr.is_empty(), "unexpected empty region");
130 // convert address range into offset range
131 _bm.at_put_range(heapWordToOffset(mr.start()),
132 heapWordToOffset(mr.end()), false);
133 }
134
135 G1CMMarkStack::G1CMMarkStack() :
136 _max_chunk_capacity(0),
137 _base(NULL),
138 _chunk_capacity(0) {
139 set_empty();
140 }
141
142 bool G1CMMarkStack::resize(size_t new_capacity) {
143 assert(is_empty(), "Only resize when stack is empty.");
144 assert(new_capacity <= _max_chunk_capacity,
145 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
146
147 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(new_capacity, mtGC);
148
149 if (new_base == NULL) {
150 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
151 return false;
152 }
421 _cleanup_times(),
422 _total_counting_time(0.0),
423 _total_rs_scrub_time(0.0),
424
425 _parallel_workers(NULL),
426
427 _completed_initialization(false) {
428
429 _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage);
430 _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage);
431
432 // Create & start a ConcurrentMark thread.
433 _cmThread = new ConcurrentMarkThread(this);
434 assert(cmThread() != NULL, "CM Thread should have been created");
435 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
436 if (_cmThread->osthread() == NULL) {
437 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
438 }
439
440 assert(CGC_lock != NULL, "Where's the CGC_lock?");
441 assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency");
442 assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency");
443
444 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
445 satb_qs.set_buffer_size(G1SATBBufferSize);
446
447 _root_regions.init(_g1h->survivor(), this);
448
449 if (ConcGCThreads > ParallelGCThreads) {
450 log_warning(gc)("Can't have more ConcGCThreads (%u) than ParallelGCThreads (%u).",
451 ConcGCThreads, ParallelGCThreads);
452 return;
453 }
454 if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) {
455 // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent
456 // if both are set
457 _sleep_factor = 0.0;
458 _marking_task_overhead = 1.0;
459 } else if (G1MarkingOverheadPercent > 0) {
460 // We will calculate the number of parallel marking threads based
461 // on a target overhead with respect to the soft real-time goal
462 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
736 // this time no other cycle can start. So, let's make sure that this
737 // is the case.
738 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
739
740 clear_bitmap(_nextMarkBitMap, _parallel_workers, true);
741
742 // Clear the live count data. If the marking has been aborted, the abort()
743 // call already did that.
744 if (!has_aborted()) {
745 clear_live_data(_parallel_workers);
746 DEBUG_ONLY(verify_live_data_clear());
747 }
748
749 // Repeat the asserts from above.
750 guarantee(cmThread()->during_cycle(), "invariant");
751 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
752 }
753
754 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
755 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
756 clear_bitmap((G1CMBitMap*)_prevMarkBitMap, workers, false);
757 }
758
759 class CheckBitmapClearHRClosure : public HeapRegionClosure {
760 G1CMBitMap* _bitmap;
761 bool _error;
762 public:
763 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
764 }
765
766 virtual bool doHeapRegion(HeapRegion* r) {
767 // This closure can be called concurrently to the mutator, so we must make sure
768 // that the result of the getNextMarkedWordAddress() call is compared to the
769 // value passed to it as limit to detect any found bits.
770 // end never changes in G1.
771 HeapWord* end = r->end();
772 return _bitmap->getNextMarkedWordAddress(r->bottom(), end) != end;
773 }
774 };
775
776 bool G1ConcurrentMark::nextMarkBitmapIsClear() {
777 CheckBitmapClearHRClosure cl(_nextMarkBitMap);
778 _g1h->heap_region_iterate(&cl);
779 return cl.complete();
780 }
781
782 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
783 public:
784 bool doHeapRegion(HeapRegion* r) {
785 r->note_start_of_marking();
786 return false;
787 }
788 };
789
790 void G1ConcurrentMark::checkpointRootsInitialPre() {
791 G1CollectedHeap* g1h = G1CollectedHeap::heap();
792 G1Policy* g1p = g1h->g1_policy();
793
794 _has_aborted = false;
795
796 // Initialize marking structures. This has to be done in a STW phase.
797 reset();
798
799 // For each region note start of marking.
800 NoteStartOfMarkHRClosure startcl;
801 g1h->heap_region_iterate(&startcl);
802 }
803
804
805 void G1ConcurrentMark::checkpointRootsInitialPost() {
806 G1CollectedHeap* g1h = G1CollectedHeap::heap();
807
808 // Start Concurrent Marking weak-reference discovery.
809 ReferenceProcessor* rp = g1h->ref_processor_cm();
810 // enable ("weak") refs discovery
811 rp->enable_discovery();
812 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
1747 return;
1748 }
1749
1750 assert(_global_mark_stack.is_empty(), "Marking should have completed");
1751
1752 // Unload Klasses, String, Symbols, Code Cache, etc.
1753 if (ClassUnloadingWithConcurrentMark) {
1754 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm);
1755 bool purged_classes = SystemDictionary::do_unloading(&g1_is_alive, _gc_timer_cm, false /* Defer cleaning */);
1756 g1h->complete_cleaning(&g1_is_alive, purged_classes);
1757 } else {
1758 GCTraceTime(Debug, gc, phases) debug("Cleanup", _gc_timer_cm);
1759 // No need to clean string table and symbol table as they are treated as strong roots when
1760 // class unloading is disabled.
1761 g1h->partial_cleaning(&g1_is_alive, false, false, G1StringDedup::is_enabled());
1762
1763 }
1764 }
1765
1766 void G1ConcurrentMark::swapMarkBitMaps() {
1767 G1CMBitMapRO* temp = _prevMarkBitMap;
1768 _prevMarkBitMap = (G1CMBitMapRO*)_nextMarkBitMap;
1769 _nextMarkBitMap = (G1CMBitMap*) temp;
1770 }
1771
1772 // Closure for marking entries in SATB buffers.
1773 class G1CMSATBBufferClosure : public SATBBufferClosure {
1774 private:
1775 G1CMTask* _task;
1776 G1CollectedHeap* _g1h;
1777
1778 // This is very similar to G1CMTask::deal_with_reference, but with
1779 // more relaxed requirements for the argument, so this must be more
1780 // circumspect about treating the argument as an object.
1781 void do_entry(void* entry) const {
1782 _task->increment_refs_reached();
1783 HeapRegion* hr = _g1h->heap_region_containing(entry);
1784 if (entry < hr->next_top_at_mark_start()) {
1785 // Until we get here, we don't know whether entry refers to a valid
1786 // object; it could instead have been a stale reference.
1787 oop obj = static_cast<oop>(entry);
1788 assert(obj->is_oop(true /* ignore mark word */),
1789 "Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj));
1894 StrongRootsScope srs(active_workers);
1895
1896 G1CMRemarkTask remarkTask(this, active_workers);
1897 // We will start all available threads, even if we decide that the
1898 // active_workers will be fewer. The extra ones will just bail out
1899 // immediately.
1900 g1h->workers()->run_task(&remarkTask);
1901 }
1902
1903 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1904 guarantee(has_overflown() ||
1905 satb_mq_set.completed_buffers_num() == 0,
1906 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT,
1907 BOOL_TO_STR(has_overflown()),
1908 satb_mq_set.completed_buffers_num());
1909
1910 print_stats();
1911 }
1912
1913 void G1ConcurrentMark::clearRangePrevBitmap(MemRegion mr) {
1914 // Note we are overriding the read-only view of the prev map here, via
1915 // the cast.
1916 ((G1CMBitMap*)_prevMarkBitMap)->clear_range(mr);
1917 }
1918
1919 HeapRegion*
1920 G1ConcurrentMark::claim_region(uint worker_id) {
1921 // "checkpoint" the finger
1922 HeapWord* finger = _finger;
1923
1924 // _heap_end will not change underneath our feet; it only changes at
1925 // yield points.
1926 while (finger < _heap_end) {
1927 assert(_g1h->is_in_g1_reserved(finger), "invariant");
1928
1929 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
1930 // Make sure that the reads below do not float before loading curr_region.
1931 OrderAccess::loadload();
1932 // Above heap_region_containing may return NULL as we always scan claim
1933 // until the end of the heap. In this case, just jump to the next region.
1934 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords;
1935
1936 // Is the gap between reading the finger and doing the CAS too long?
2143 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2144 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2145 cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());
2146 }
2147
2148 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2149 _parallel_workers->print_worker_threads_on(st);
2150 }
2151
2152 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
2153 _parallel_workers->threads_do(tc);
2154 }
2155
2156 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2157 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2158 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2159 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2160 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2161 }
2162
2163 // Closure for iteration over bitmaps
2164 class G1CMBitMapClosure : public BitMapClosure {
2165 private:
2166 // the bitmap that is being iterated over
2167 G1CMBitMap* _nextMarkBitMap;
2168 G1ConcurrentMark* _cm;
2169 G1CMTask* _task;
2170
2171 public:
2172 G1CMBitMapClosure(G1CMTask *task, G1ConcurrentMark* cm, G1CMBitMap* nextMarkBitMap) :
2173 _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { }
2174
2175 bool do_bit(size_t offset) {
2176 HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset);
2177 assert(_nextMarkBitMap->isMarked(addr), "invariant");
2178 assert( addr < _cm->finger(), "invariant");
2179 assert(addr >= _task->finger(), "invariant");
2180
2181 // We move that task's local finger along.
2182 _task->move_finger_to(addr);
2183
2184 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
2185 // we only partially drain the local queue and global stack
2186 _task->drain_local_queue(true);
2187 _task->drain_global_stack(true);
2188
2189 // if the has_aborted flag has been raised, we need to bail out of
2190 // the iteration
2191 return !_task->has_aborted();
2192 }
2193 };
2194
2195 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
2196 ReferenceProcessor* result = g1h->ref_processor_cm();
2197 assert(result != NULL, "CM reference processor should not be NULL");
2198 return result;
2199 }
2200
2201 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
2202 G1ConcurrentMark* cm,
2203 G1CMTask* task)
2204 : MetadataAwareOopClosure(get_cm_oop_closure_ref_processor(g1h)),
2205 _g1h(g1h), _cm(cm), _task(task)
2206 { }
2207
2208 void G1CMTask::setup_for_region(HeapRegion* hr) {
2209 assert(hr != NULL,
2210 "claim_region() should have filtered out NULL regions");
2211 _curr_region = hr;
2212 _finger = hr->bottom();
2213 update_region_limit();
2674
2675 double diff_prediction_ms = _g1h->g1_policy()->predictor().get_new_prediction(&_marking_step_diffs_ms);
2676 _time_target_ms = time_target_ms - diff_prediction_ms;
2677
2678 // set up the variables that are used in the work-based scheme to
2679 // call the regular clock method
2680 _words_scanned = 0;
2681 _refs_reached = 0;
2682 recalculate_limits();
2683
2684 // clear all flags
2685 clear_has_aborted();
2686 _has_timed_out = false;
2687 _draining_satb_buffers = false;
2688
2689 ++_calls;
2690
2691 // Set up the bitmap and oop closures. Anything that uses them is
2692 // eventually called from this method, so it is OK to allocate these
2693 // statically.
2694 G1CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap);
2695 G1CMOopClosure cm_oop_closure(_g1h, _cm, this);
2696 set_cm_oop_closure(&cm_oop_closure);
2697
2698 if (_cm->has_overflown()) {
2699 // This can happen if the mark stack overflows during a GC pause
2700 // and this task, after a yield point, restarts. We have to abort
2701 // as we need to get into the overflow protocol which happens
2702 // right at the end of this task.
2703 set_has_aborted();
2704 }
2705
2706 // First drain any available SATB buffers. After this, we will not
2707 // look at SATB buffers before the next invocation of this method.
2708 // If enough completed SATB buffers are queued up, the regular clock
2709 // will abort this task so that it restarts.
2710 drain_satb_buffers();
2711 // ...then partially drain the local queue and the global stack
2712 drain_local_queue(true);
2713 drain_global_stack(true);
2714
2730 // through scanning this region. In this case, _finger points to
2731 // the address where we last found a marked object. If this is a
2732 // fresh region, _finger points to start().
2733 MemRegion mr = MemRegion(_finger, _region_limit);
2734
2735 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
2736 "humongous regions should go around loop once only");
2737
2738 // Some special cases:
2739 // If the memory region is empty, we can just give up the region.
2740 // If the current region is humongous then we only need to check
2741 // the bitmap for the bit associated with the start of the object,
2742 // scan the object if it's live, and give up the region.
2743 // Otherwise, let's iterate over the bitmap of the part of the region
2744 // that is left.
2745 // If the iteration is successful, give up the region.
2746 if (mr.is_empty()) {
2747 giveup_current_region();
2748 regular_clock_call();
2749 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
2750 if (_nextMarkBitMap->isMarked(mr.start())) {
2751 // The object is marked - apply the closure
2752 BitMap::idx_t offset = _nextMarkBitMap->heapWordToOffset(mr.start());
2753 bitmap_closure.do_bit(offset);
2754 }
2755 // Even if this task aborted while scanning the humongous object
2756 // we can (and should) give up the current region.
2757 giveup_current_region();
2758 regular_clock_call();
2759 } else if (_nextMarkBitMap->iterate(&bitmap_closure, mr)) {
2760 giveup_current_region();
2761 regular_clock_call();
2762 } else {
2763 assert(has_aborted(), "currently the only way to do so");
2764 // The only way to abort the bitmap iteration is to return
2765 // false from the do_bit() method. However, inside the
2766 // do_bit() method we move the _finger to point to the
2767 // object currently being looked at. So, if we bail out, we
2768 // have definitely set _finger to something non-null.
2769 assert(_finger != NULL, "invariant");
2770
2771 // Region iteration was actually aborted. So now _finger
2772 // points to the address of the object we last scanned. If we
2773 // leave it there, when we restart this task, we will rescan
2774 // the object. It is easy to avoid this. We move the finger by
2775 // enough to point to the next possible object header (the
2776 // bitmap knows by how much we need to move it as it knows its
2777 // granularity).
2778 assert(_finger < _region_limit, "invariant");
2779 HeapWord* new_finger = _nextMarkBitMap->nextObject(_finger);
2780 // Check if bitmap iteration was aborted while scanning the last object
2781 if (new_finger >= _region_limit) {
2782 giveup_current_region();
2783 } else {
2784 move_finger_to(new_finger);
2785 }
2786 }
2787 }
2788 // At this point we have either completed iterating over the
2789 // region we were holding on to, or we have aborted.
2790
2791 // We then partially drain the local queue and the global stack.
2792 // (Do we really need this?)
2793 drain_local_queue(true);
2794 drain_global_stack(true);
2795
2796 // Read the note on the claim_region() method on why it might
2797 // return NULL with potentially more regions available for
2798 // claiming and why we have to check out_of_regions() to determine
2799 // whether we're done or not.
|
43 #include "gc/shared/gcTimer.hpp"
44 #include "gc/shared/gcTrace.hpp"
45 #include "gc/shared/gcTraceTime.inline.hpp"
46 #include "gc/shared/genOopClosures.inline.hpp"
47 #include "gc/shared/referencePolicy.hpp"
48 #include "gc/shared/strongRootsScope.hpp"
49 #include "gc/shared/taskqueue.inline.hpp"
50 #include "gc/shared/vmGCOperations.hpp"
51 #include "logging/log.hpp"
52 #include "memory/allocation.hpp"
53 #include "memory/resourceArea.hpp"
54 #include "oops/oop.inline.hpp"
55 #include "runtime/atomic.hpp"
56 #include "runtime/handles.inline.hpp"
57 #include "runtime/java.hpp"
58 #include "runtime/prefetch.inline.hpp"
59 #include "services/memTracker.hpp"
60 #include "utilities/align.hpp"
61 #include "utilities/growableArray.hpp"
62
63 void G1CMBitMap::print_on_error(outputStream* st, const char* prefix) const {
64 _bm.print_on_error(st, prefix);
65 }
66
67 size_t G1CMBitMap::compute_size(size_t heap_size) {
68 return ReservedSpace::allocation_align_size_up(heap_size / mark_distance());
69 }
70
71 size_t G1CMBitMap::mark_distance() {
72 return MinObjAlignmentInBytes * BitsPerByte;
73 }
74
75 void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) {
76 _covered = heap;
77
78 _bm = BitMapView((BitMap::bm_word_t*) storage->reserved().start(), _covered.word_size() >> _shifter);
79
80 storage->set_mapping_changed_listener(&_listener);
81 }
82
83 void G1CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) {
84 if (zero_filled) {
85 return;
86 }
87 // We need to clear the bitmap on commit, removing any existing information.
88 MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords);
89 _bm->clear_range(mr);
90 }
91
92 void G1CMBitMap::clear_range(MemRegion mr) {
93 MemRegion intersection = mr.intersection(_covered);
94 assert(!intersection.is_empty(), "Given range from " PTR_FORMAT " to " PTR_FORMAT " is completely outside the heap", p2i(mr.start()), p2i(mr.end()));
95 // convert address range into offset range
96 _bm.at_put_range(addr_to_offset(intersection.start()),
97 addr_to_offset(intersection.end()), false);
98 }
99
100 G1CMMarkStack::G1CMMarkStack() :
101 _max_chunk_capacity(0),
102 _base(NULL),
103 _chunk_capacity(0) {
104 set_empty();
105 }
106
107 bool G1CMMarkStack::resize(size_t new_capacity) {
108 assert(is_empty(), "Only resize when stack is empty.");
109 assert(new_capacity <= _max_chunk_capacity,
110 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
111
112 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(new_capacity, mtGC);
113
114 if (new_base == NULL) {
115 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
116 return false;
117 }
386 _cleanup_times(),
387 _total_counting_time(0.0),
388 _total_rs_scrub_time(0.0),
389
390 _parallel_workers(NULL),
391
392 _completed_initialization(false) {
393
394 _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage);
395 _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage);
396
397 // Create & start a ConcurrentMark thread.
398 _cmThread = new ConcurrentMarkThread(this);
399 assert(cmThread() != NULL, "CM Thread should have been created");
400 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
401 if (_cmThread->osthread() == NULL) {
402 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
403 }
404
405 assert(CGC_lock != NULL, "Where's the CGC_lock?");
406
407 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
408 satb_qs.set_buffer_size(G1SATBBufferSize);
409
410 _root_regions.init(_g1h->survivor(), this);
411
412 if (ConcGCThreads > ParallelGCThreads) {
413 log_warning(gc)("Can't have more ConcGCThreads (%u) than ParallelGCThreads (%u).",
414 ConcGCThreads, ParallelGCThreads);
415 return;
416 }
417 if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) {
418 // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent
419 // if both are set
420 _sleep_factor = 0.0;
421 _marking_task_overhead = 1.0;
422 } else if (G1MarkingOverheadPercent > 0) {
423 // We will calculate the number of parallel marking threads based
424 // on a target overhead with respect to the soft real-time goal
425 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
699 // this time no other cycle can start. So, let's make sure that this
700 // is the case.
701 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
702
703 clear_bitmap(_nextMarkBitMap, _parallel_workers, true);
704
705 // Clear the live count data. If the marking has been aborted, the abort()
706 // call already did that.
707 if (!has_aborted()) {
708 clear_live_data(_parallel_workers);
709 DEBUG_ONLY(verify_live_data_clear());
710 }
711
712 // Repeat the asserts from above.
713 guarantee(cmThread()->during_cycle(), "invariant");
714 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
715 }
716
717 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
718 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
719 clear_bitmap(_prevMarkBitMap, workers, false);
720 }
721
722 class CheckBitmapClearHRClosure : public HeapRegionClosure {
723 G1CMBitMap* _bitmap;
724 bool _error;
725 public:
726 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
727 }
728
729 virtual bool doHeapRegion(HeapRegion* r) {
730 // This closure can be called concurrently to the mutator, so we must make sure
731 // that the result of the getNextMarkedWordAddress() call is compared to the
732 // value passed to it as limit to detect any found bits.
733 // end never changes in G1.
734 HeapWord* end = r->end();
735 return _bitmap->get_next_marked_addr(r->bottom(), end) != end;
736 }
737 };
738
739 bool G1ConcurrentMark::nextMarkBitmapIsClear() {
740 CheckBitmapClearHRClosure cl(_nextMarkBitMap);
741 _g1h->heap_region_iterate(&cl);
742 return cl.complete();
743 }
744
745 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
746 public:
747 bool doHeapRegion(HeapRegion* r) {
748 r->note_start_of_marking();
749 return false;
750 }
751 };
752
753 void G1ConcurrentMark::checkpointRootsInitialPre() {
754 G1CollectedHeap* g1h = G1CollectedHeap::heap();
755
756 _has_aborted = false;
757
758 // Initialize marking structures. This has to be done in a STW phase.
759 reset();
760
761 // For each region note start of marking.
762 NoteStartOfMarkHRClosure startcl;
763 g1h->heap_region_iterate(&startcl);
764 }
765
766
767 void G1ConcurrentMark::checkpointRootsInitialPost() {
768 G1CollectedHeap* g1h = G1CollectedHeap::heap();
769
770 // Start Concurrent Marking weak-reference discovery.
771 ReferenceProcessor* rp = g1h->ref_processor_cm();
772 // enable ("weak") refs discovery
773 rp->enable_discovery();
774 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
1709 return;
1710 }
1711
1712 assert(_global_mark_stack.is_empty(), "Marking should have completed");
1713
1714 // Unload Klasses, String, Symbols, Code Cache, etc.
1715 if (ClassUnloadingWithConcurrentMark) {
1716 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm);
1717 bool purged_classes = SystemDictionary::do_unloading(&g1_is_alive, _gc_timer_cm, false /* Defer cleaning */);
1718 g1h->complete_cleaning(&g1_is_alive, purged_classes);
1719 } else {
1720 GCTraceTime(Debug, gc, phases) debug("Cleanup", _gc_timer_cm);
1721 // No need to clean string table and symbol table as they are treated as strong roots when
1722 // class unloading is disabled.
1723 g1h->partial_cleaning(&g1_is_alive, false, false, G1StringDedup::is_enabled());
1724
1725 }
1726 }
1727
1728 void G1ConcurrentMark::swapMarkBitMaps() {
1729 G1CMBitMap* temp = _prevMarkBitMap;
1730 _prevMarkBitMap = _nextMarkBitMap;
1731 _nextMarkBitMap = temp;
1732 }
1733
1734 // Closure for marking entries in SATB buffers.
1735 class G1CMSATBBufferClosure : public SATBBufferClosure {
1736 private:
1737 G1CMTask* _task;
1738 G1CollectedHeap* _g1h;
1739
1740 // This is very similar to G1CMTask::deal_with_reference, but with
1741 // more relaxed requirements for the argument, so this must be more
1742 // circumspect about treating the argument as an object.
1743 void do_entry(void* entry) const {
1744 _task->increment_refs_reached();
1745 HeapRegion* hr = _g1h->heap_region_containing(entry);
1746 if (entry < hr->next_top_at_mark_start()) {
1747 // Until we get here, we don't know whether entry refers to a valid
1748 // object; it could instead have been a stale reference.
1749 oop obj = static_cast<oop>(entry);
1750 assert(obj->is_oop(true /* ignore mark word */),
1751 "Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj));
1856 StrongRootsScope srs(active_workers);
1857
1858 G1CMRemarkTask remarkTask(this, active_workers);
1859 // We will start all available threads, even if we decide that the
1860 // active_workers will be fewer. The extra ones will just bail out
1861 // immediately.
1862 g1h->workers()->run_task(&remarkTask);
1863 }
1864
1865 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1866 guarantee(has_overflown() ||
1867 satb_mq_set.completed_buffers_num() == 0,
1868 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT,
1869 BOOL_TO_STR(has_overflown()),
1870 satb_mq_set.completed_buffers_num());
1871
1872 print_stats();
1873 }
1874
1875 void G1ConcurrentMark::clearRangePrevBitmap(MemRegion mr) {
1876 _prevMarkBitMap->clear_range(mr);
1877 }
1878
1879 HeapRegion*
1880 G1ConcurrentMark::claim_region(uint worker_id) {
1881 // "checkpoint" the finger
1882 HeapWord* finger = _finger;
1883
1884 // _heap_end will not change underneath our feet; it only changes at
1885 // yield points.
1886 while (finger < _heap_end) {
1887 assert(_g1h->is_in_g1_reserved(finger), "invariant");
1888
1889 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
1890 // Make sure that the reads below do not float before loading curr_region.
1891 OrderAccess::loadload();
1892 // Above heap_region_containing may return NULL as we always scan claim
1893 // until the end of the heap. In this case, just jump to the next region.
1894 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords;
1895
1896 // Is the gap between reading the finger and doing the CAS too long?
2103 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2104 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2105 cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());
2106 }
2107
2108 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2109 _parallel_workers->print_worker_threads_on(st);
2110 }
2111
2112 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
2113 _parallel_workers->threads_do(tc);
2114 }
2115
2116 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2117 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2118 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2119 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2120 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2121 }
2122
2123 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
2124 assert(addr < _cm->finger(), "invariant");
2125 assert(addr >= _task->finger(), "invariant");
2126
2127 // We move that task's local finger along.
2128 _task->move_finger_to(addr);
2129
2130 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
2131 // we only partially drain the local queue and global stack
2132 _task->drain_local_queue(true);
2133 _task->drain_global_stack(true);
2134
2135 // if the has_aborted flag has been raised, we need to bail out of
2136 // the iteration
2137 return !_task->has_aborted();
2138 }
2139
2140 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
2141 ReferenceProcessor* result = g1h->ref_processor_cm();
2142 assert(result != NULL, "CM reference processor should not be NULL");
2143 return result;
2144 }
2145
2146 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
2147 G1ConcurrentMark* cm,
2148 G1CMTask* task)
2149 : MetadataAwareOopClosure(get_cm_oop_closure_ref_processor(g1h)),
2150 _g1h(g1h), _cm(cm), _task(task)
2151 { }
2152
2153 void G1CMTask::setup_for_region(HeapRegion* hr) {
2154 assert(hr != NULL,
2155 "claim_region() should have filtered out NULL regions");
2156 _curr_region = hr;
2157 _finger = hr->bottom();
2158 update_region_limit();
2619
2620 double diff_prediction_ms = _g1h->g1_policy()->predictor().get_new_prediction(&_marking_step_diffs_ms);
2621 _time_target_ms = time_target_ms - diff_prediction_ms;
2622
2623 // set up the variables that are used in the work-based scheme to
2624 // call the regular clock method
2625 _words_scanned = 0;
2626 _refs_reached = 0;
2627 recalculate_limits();
2628
2629 // clear all flags
2630 clear_has_aborted();
2631 _has_timed_out = false;
2632 _draining_satb_buffers = false;
2633
2634 ++_calls;
2635
2636 // Set up the bitmap and oop closures. Anything that uses them is
2637 // eventually called from this method, so it is OK to allocate these
2638 // statically.
2639 G1CMBitMapClosure bitmap_closure(this, _cm);
2640 G1CMOopClosure cm_oop_closure(_g1h, _cm, this);
2641 set_cm_oop_closure(&cm_oop_closure);
2642
2643 if (_cm->has_overflown()) {
2644 // This can happen if the mark stack overflows during a GC pause
2645 // and this task, after a yield point, restarts. We have to abort
2646 // as we need to get into the overflow protocol which happens
2647 // right at the end of this task.
2648 set_has_aborted();
2649 }
2650
2651 // First drain any available SATB buffers. After this, we will not
2652 // look at SATB buffers before the next invocation of this method.
2653 // If enough completed SATB buffers are queued up, the regular clock
2654 // will abort this task so that it restarts.
2655 drain_satb_buffers();
2656 // ...then partially drain the local queue and the global stack
2657 drain_local_queue(true);
2658 drain_global_stack(true);
2659
2675 // through scanning this region. In this case, _finger points to
2676 // the address where we last found a marked object. If this is a
2677 // fresh region, _finger points to start().
2678 MemRegion mr = MemRegion(_finger, _region_limit);
2679
2680 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
2681 "humongous regions should go around loop once only");
2682
2683 // Some special cases:
2684 // If the memory region is empty, we can just give up the region.
2685 // If the current region is humongous then we only need to check
2686 // the bitmap for the bit associated with the start of the object,
2687 // scan the object if it's live, and give up the region.
2688 // Otherwise, let's iterate over the bitmap of the part of the region
2689 // that is left.
2690 // If the iteration is successful, give up the region.
2691 if (mr.is_empty()) {
2692 giveup_current_region();
2693 regular_clock_call();
2694 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
2695 if (_nextMarkBitMap->is_marked(mr.start())) {
2696 // The object is marked - apply the closure
2697 bitmap_closure.do_addr(mr.start());
2698 }
2699 // Even if this task aborted while scanning the humongous object
2700 // we can (and should) give up the current region.
2701 giveup_current_region();
2702 regular_clock_call();
2703 } else if (_nextMarkBitMap->iterate(&bitmap_closure, mr)) {
2704 giveup_current_region();
2705 regular_clock_call();
2706 } else {
2707 assert(has_aborted(), "currently the only way to do so");
2708 // The only way to abort the bitmap iteration is to return
2709 // false from the do_bit() method. However, inside the
2710 // do_bit() method we move the _finger to point to the
2711 // object currently being looked at. So, if we bail out, we
2712 // have definitely set _finger to something non-null.
2713 assert(_finger != NULL, "invariant");
2714
2715 // Region iteration was actually aborted. So now _finger
2716 // points to the address of the object we last scanned. If we
2717 // leave it there, when we restart this task, we will rescan
2718 // the object. It is easy to avoid this. We move the finger by
2719 // enough to point to the next possible object header.
2720 assert(_finger < _region_limit, "invariant");
2721 HeapWord* const new_finger = _finger + ((oop)_finger)->size();
2722 // Check if bitmap iteration was aborted while scanning the last object
2723 if (new_finger >= _region_limit) {
2724 giveup_current_region();
2725 } else {
2726 move_finger_to(new_finger);
2727 }
2728 }
2729 }
2730 // At this point we have either completed iterating over the
2731 // region we were holding on to, or we have aborted.
2732
2733 // We then partially drain the local queue and the global stack.
2734 // (Do we really need this?)
2735 drain_local_queue(true);
2736 drain_global_stack(true);
2737
2738 // Read the note on the claim_region() method on why it might
2739 // return NULL with potentially more regions available for
2740 // claiming and why we have to check out_of_regions() to determine
2741 // whether we're done or not.
|