43 #include "gc/shared/gcTimer.hpp"
44 #include "gc/shared/gcTrace.hpp"
45 #include "gc/shared/gcTraceTime.inline.hpp"
46 #include "gc/shared/genOopClosures.inline.hpp"
47 #include "gc/shared/referencePolicy.hpp"
48 #include "gc/shared/strongRootsScope.hpp"
49 #include "gc/shared/taskqueue.inline.hpp"
50 #include "gc/shared/vmGCOperations.hpp"
51 #include "logging/log.hpp"
52 #include "memory/allocation.hpp"
53 #include "memory/resourceArea.hpp"
54 #include "oops/oop.inline.hpp"
55 #include "runtime/atomic.hpp"
56 #include "runtime/handles.inline.hpp"
57 #include "runtime/java.hpp"
58 #include "runtime/prefetch.inline.hpp"
59 #include "services/memTracker.hpp"
60 #include "utilities/align.hpp"
61 #include "utilities/growableArray.hpp"
62
63 // Concurrent marking bit map wrapper
64
65 G1CMBitMapRO::G1CMBitMapRO(int shifter) :
66 _bm(),
67 _shifter(shifter) {
68 _bmStartWord = 0;
69 _bmWordSize = 0;
70 }
71
72 HeapWord* G1CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
73 const HeapWord* limit) const {
74 // First we must round addr *up* to a possible object boundary.
75 addr = align_up(addr, HeapWordSize << _shifter);
76 size_t addrOffset = heapWordToOffset(addr);
77 assert(limit != NULL, "limit must not be NULL");
78 size_t limitOffset = heapWordToOffset(limit);
79 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
80 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
81 assert(nextAddr >= addr, "get_next_one postcondition");
82 assert(nextAddr == limit || isMarked(nextAddr),
83 "get_next_one postcondition");
84 return nextAddr;
85 }
86
87 #ifndef PRODUCT
88 bool G1CMBitMapRO::covers(MemRegion heap_rs) const {
89 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
90 assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize,
91 "size inconsistency");
92 return _bmStartWord == (HeapWord*)(heap_rs.start()) &&
93 _bmWordSize == heap_rs.word_size();
94 }
95 #endif
96
97 void G1CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const {
98 _bm.print_on_error(st, prefix);
99 }
100
101 size_t G1CMBitMap::compute_size(size_t heap_size) {
102 return ReservedSpace::allocation_align_size_up(heap_size / mark_distance());
103 }
104
105 size_t G1CMBitMap::mark_distance() {
106 return MinObjAlignmentInBytes * BitsPerByte;
107 }
108
109 void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) {
110 _bmStartWord = heap.start();
111 _bmWordSize = heap.word_size();
112
113 _bm = BitMapView((BitMap::bm_word_t*) storage->reserved().start(), _bmWordSize >> _shifter);
114
115 storage->set_mapping_changed_listener(&_listener);
116 }
117
118 void G1CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) {
119 if (zero_filled) {
120 return;
121 }
122 // We need to clear the bitmap on commit, removing any existing information.
123 MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords);
124 _bm->clear_range(mr);
125 }
126
127 void G1CMBitMap::clear_range(MemRegion mr) {
128 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
129 assert(!mr.is_empty(), "unexpected empty region");
130 // convert address range into offset range
131 _bm.at_put_range(heapWordToOffset(mr.start()),
132 heapWordToOffset(mr.end()), false);
133 }
134
135 G1CMMarkStack::G1CMMarkStack() :
136 _max_chunk_capacity(0),
137 _base(NULL),
138 _chunk_capacity(0) {
139 set_empty();
140 }
141
142 bool G1CMMarkStack::resize(size_t new_capacity) {
143 assert(is_empty(), "Only resize when stack is empty.");
144 assert(new_capacity <= _max_chunk_capacity,
145 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
146
147 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk, mtGC>::allocate_or_null(new_capacity);
148
149 if (new_base == NULL) {
150 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
151 return false;
152 }
421 _cleanup_times(),
422 _total_counting_time(0.0),
423 _total_rs_scrub_time(0.0),
424
425 _parallel_workers(NULL),
426
427 _completed_initialization(false) {
428
429 _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage);
430 _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage);
431
432 // Create & start a ConcurrentMark thread.
433 _cmThread = new ConcurrentMarkThread(this);
434 assert(cmThread() != NULL, "CM Thread should have been created");
435 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
436 if (_cmThread->osthread() == NULL) {
437 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
438 }
439
440 assert(CGC_lock != NULL, "Where's the CGC_lock?");
441 assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency");
442 assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency");
443
444 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
445 satb_qs.set_buffer_size(G1SATBBufferSize);
446
447 _root_regions.init(_g1h->survivor(), this);
448
449 if (ConcGCThreads > ParallelGCThreads) {
450 log_warning(gc)("Can't have more ConcGCThreads (%u) than ParallelGCThreads (%u).",
451 ConcGCThreads, ParallelGCThreads);
452 return;
453 }
454 if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) {
455 // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent
456 // if both are set
457 _sleep_factor = 0.0;
458 _marking_task_overhead = 1.0;
459 } else if (G1MarkingOverheadPercent > 0) {
460 // We will calculate the number of parallel marking threads based
461 // on a target overhead with respect to the soft real-time goal
462 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
736 // this time no other cycle can start. So, let's make sure that this
737 // is the case.
738 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
739
740 clear_bitmap(_nextMarkBitMap, _parallel_workers, true);
741
742 // Clear the live count data. If the marking has been aborted, the abort()
743 // call already did that.
744 if (!has_aborted()) {
745 clear_live_data(_parallel_workers);
746 DEBUG_ONLY(verify_live_data_clear());
747 }
748
749 // Repeat the asserts from above.
750 guarantee(cmThread()->during_cycle(), "invariant");
751 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
752 }
753
754 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
755 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
756 clear_bitmap((G1CMBitMap*)_prevMarkBitMap, workers, false);
757 }
758
759 class CheckBitmapClearHRClosure : public HeapRegionClosure {
760 G1CMBitMap* _bitmap;
761 bool _error;
762 public:
763 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
764 }
765
766 virtual bool doHeapRegion(HeapRegion* r) {
767 // This closure can be called concurrently to the mutator, so we must make sure
768 // that the result of the getNextMarkedWordAddress() call is compared to the
769 // value passed to it as limit to detect any found bits.
770 // end never changes in G1.
771 HeapWord* end = r->end();
772 return _bitmap->getNextMarkedWordAddress(r->bottom(), end) != end;
773 }
774 };
775
776 bool G1ConcurrentMark::nextMarkBitmapIsClear() {
777 CheckBitmapClearHRClosure cl(_nextMarkBitMap);
778 _g1h->heap_region_iterate(&cl);
779 return cl.complete();
780 }
781
782 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
783 public:
784 bool doHeapRegion(HeapRegion* r) {
785 r->note_start_of_marking();
786 return false;
787 }
788 };
789
790 void G1ConcurrentMark::checkpointRootsInitialPre() {
791 G1CollectedHeap* g1h = G1CollectedHeap::heap();
792 G1Policy* g1p = g1h->g1_policy();
793
794 _has_aborted = false;
795
796 // Initialize marking structures. This has to be done in a STW phase.
797 reset();
798
799 // For each region note start of marking.
800 NoteStartOfMarkHRClosure startcl;
801 g1h->heap_region_iterate(&startcl);
802 }
803
804
805 void G1ConcurrentMark::checkpointRootsInitialPost() {
806 G1CollectedHeap* g1h = G1CollectedHeap::heap();
807
808 // Start Concurrent Marking weak-reference discovery.
809 ReferenceProcessor* rp = g1h->ref_processor_cm();
810 // enable ("weak") refs discovery
811 rp->enable_discovery();
812 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
1747 return;
1748 }
1749
1750 assert(_global_mark_stack.is_empty(), "Marking should have completed");
1751
1752 // Unload Klasses, String, Symbols, Code Cache, etc.
1753 if (ClassUnloadingWithConcurrentMark) {
1754 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm);
1755 bool purged_classes = SystemDictionary::do_unloading(&g1_is_alive, _gc_timer_cm, false /* Defer cleaning */);
1756 g1h->complete_cleaning(&g1_is_alive, purged_classes);
1757 } else {
1758 GCTraceTime(Debug, gc, phases) debug("Cleanup", _gc_timer_cm);
1759 // No need to clean string table and symbol table as they are treated as strong roots when
1760 // class unloading is disabled.
1761 g1h->partial_cleaning(&g1_is_alive, false, false, G1StringDedup::is_enabled());
1762
1763 }
1764 }
1765
1766 void G1ConcurrentMark::swapMarkBitMaps() {
1767 G1CMBitMapRO* temp = _prevMarkBitMap;
1768 _prevMarkBitMap = (G1CMBitMapRO*)_nextMarkBitMap;
1769 _nextMarkBitMap = (G1CMBitMap*) temp;
1770 }
1771
1772 // Closure for marking entries in SATB buffers.
1773 class G1CMSATBBufferClosure : public SATBBufferClosure {
1774 private:
1775 G1CMTask* _task;
1776 G1CollectedHeap* _g1h;
1777
1778 // This is very similar to G1CMTask::deal_with_reference, but with
1779 // more relaxed requirements for the argument, so this must be more
1780 // circumspect about treating the argument as an object.
1781 void do_entry(void* entry) const {
1782 _task->increment_refs_reached();
1783 HeapRegion* hr = _g1h->heap_region_containing(entry);
1784 if (entry < hr->next_top_at_mark_start()) {
1785 // Until we get here, we don't know whether entry refers to a valid
1786 // object; it could instead have been a stale reference.
1787 oop obj = static_cast<oop>(entry);
1788 assert(obj->is_oop(true /* ignore mark word */),
1789 "Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj));
1894 StrongRootsScope srs(active_workers);
1895
1896 G1CMRemarkTask remarkTask(this, active_workers);
1897 // We will start all available threads, even if we decide that the
1898 // active_workers will be fewer. The extra ones will just bail out
1899 // immediately.
1900 g1h->workers()->run_task(&remarkTask);
1901 }
1902
1903 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1904 guarantee(has_overflown() ||
1905 satb_mq_set.completed_buffers_num() == 0,
1906 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT,
1907 BOOL_TO_STR(has_overflown()),
1908 satb_mq_set.completed_buffers_num());
1909
1910 print_stats();
1911 }
1912
1913 void G1ConcurrentMark::clearRangePrevBitmap(MemRegion mr) {
1914 // Note we are overriding the read-only view of the prev map here, via
1915 // the cast.
1916 ((G1CMBitMap*)_prevMarkBitMap)->clear_range(mr);
1917 }
1918
1919 HeapRegion*
1920 G1ConcurrentMark::claim_region(uint worker_id) {
1921 // "checkpoint" the finger
1922 HeapWord* finger = _finger;
1923
1924 // _heap_end will not change underneath our feet; it only changes at
1925 // yield points.
1926 while (finger < _heap_end) {
1927 assert(_g1h->is_in_g1_reserved(finger), "invariant");
1928
1929 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
1930 // Make sure that the reads below do not float before loading curr_region.
1931 OrderAccess::loadload();
1932 // Above heap_region_containing may return NULL as we always scan claim
1933 // until the end of the heap. In this case, just jump to the next region.
1934 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords;
1935
1936 // Is the gap between reading the finger and doing the CAS too long?
2143 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2144 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2145 cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());
2146 }
2147
2148 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2149 _parallel_workers->print_worker_threads_on(st);
2150 }
2151
2152 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
2153 _parallel_workers->threads_do(tc);
2154 }
2155
2156 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2157 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2158 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2159 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2160 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2161 }
2162
2163 // Closure for iteration over bitmaps
2164 class G1CMBitMapClosure : public BitMapClosure {
2165 private:
2166 // the bitmap that is being iterated over
2167 G1CMBitMap* _nextMarkBitMap;
2168 G1ConcurrentMark* _cm;
2169 G1CMTask* _task;
2170
2171 public:
2172 G1CMBitMapClosure(G1CMTask *task, G1ConcurrentMark* cm, G1CMBitMap* nextMarkBitMap) :
2173 _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { }
2174
2175 bool do_bit(size_t offset) {
2176 HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset);
2177 assert(_nextMarkBitMap->isMarked(addr), "invariant");
2178 assert( addr < _cm->finger(), "invariant");
2179 assert(addr >= _task->finger(), "invariant");
2180
2181 // We move that task's local finger along.
2182 _task->move_finger_to(addr);
2183
2184 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
2185 // we only partially drain the local queue and global stack
2186 _task->drain_local_queue(true);
2187 _task->drain_global_stack(true);
2188
2189 // if the has_aborted flag has been raised, we need to bail out of
2190 // the iteration
2191 return !_task->has_aborted();
2192 }
2193 };
2194
2195 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
2196 ReferenceProcessor* result = g1h->ref_processor_cm();
2197 assert(result != NULL, "CM reference processor should not be NULL");
2198 return result;
2199 }
2200
2201 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
2202 G1ConcurrentMark* cm,
2203 G1CMTask* task)
2204 : MetadataAwareOopClosure(get_cm_oop_closure_ref_processor(g1h)),
2205 _g1h(g1h), _cm(cm), _task(task)
2206 { }
2207
2208 void G1CMTask::setup_for_region(HeapRegion* hr) {
2209 assert(hr != NULL,
2210 "claim_region() should have filtered out NULL regions");
2211 _curr_region = hr;
2212 _finger = hr->bottom();
2213 update_region_limit();
2674
2675 double diff_prediction_ms = _g1h->g1_policy()->predictor().get_new_prediction(&_marking_step_diffs_ms);
2676 _time_target_ms = time_target_ms - diff_prediction_ms;
2677
2678 // set up the variables that are used in the work-based scheme to
2679 // call the regular clock method
2680 _words_scanned = 0;
2681 _refs_reached = 0;
2682 recalculate_limits();
2683
2684 // clear all flags
2685 clear_has_aborted();
2686 _has_timed_out = false;
2687 _draining_satb_buffers = false;
2688
2689 ++_calls;
2690
2691 // Set up the bitmap and oop closures. Anything that uses them is
2692 // eventually called from this method, so it is OK to allocate these
2693 // statically.
2694 G1CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap);
2695 G1CMOopClosure cm_oop_closure(_g1h, _cm, this);
2696 set_cm_oop_closure(&cm_oop_closure);
2697
2698 if (_cm->has_overflown()) {
2699 // This can happen if the mark stack overflows during a GC pause
2700 // and this task, after a yield point, restarts. We have to abort
2701 // as we need to get into the overflow protocol which happens
2702 // right at the end of this task.
2703 set_has_aborted();
2704 }
2705
2706 // First drain any available SATB buffers. After this, we will not
2707 // look at SATB buffers before the next invocation of this method.
2708 // If enough completed SATB buffers are queued up, the regular clock
2709 // will abort this task so that it restarts.
2710 drain_satb_buffers();
2711 // ...then partially drain the local queue and the global stack
2712 drain_local_queue(true);
2713 drain_global_stack(true);
2714
2730 // through scanning this region. In this case, _finger points to
2731 // the address where we last found a marked object. If this is a
2732 // fresh region, _finger points to start().
2733 MemRegion mr = MemRegion(_finger, _region_limit);
2734
2735 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
2736 "humongous regions should go around loop once only");
2737
2738 // Some special cases:
2739 // If the memory region is empty, we can just give up the region.
2740 // If the current region is humongous then we only need to check
2741 // the bitmap for the bit associated with the start of the object,
2742 // scan the object if it's live, and give up the region.
2743 // Otherwise, let's iterate over the bitmap of the part of the region
2744 // that is left.
2745 // If the iteration is successful, give up the region.
2746 if (mr.is_empty()) {
2747 giveup_current_region();
2748 regular_clock_call();
2749 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
2750 if (_nextMarkBitMap->isMarked(mr.start())) {
2751 // The object is marked - apply the closure
2752 BitMap::idx_t offset = _nextMarkBitMap->heapWordToOffset(mr.start());
2753 bitmap_closure.do_bit(offset);
2754 }
2755 // Even if this task aborted while scanning the humongous object
2756 // we can (and should) give up the current region.
2757 giveup_current_region();
2758 regular_clock_call();
2759 } else if (_nextMarkBitMap->iterate(&bitmap_closure, mr)) {
2760 giveup_current_region();
2761 regular_clock_call();
2762 } else {
2763 assert(has_aborted(), "currently the only way to do so");
2764 // The only way to abort the bitmap iteration is to return
2765 // false from the do_bit() method. However, inside the
2766 // do_bit() method we move the _finger to point to the
2767 // object currently being looked at. So, if we bail out, we
2768 // have definitely set _finger to something non-null.
2769 assert(_finger != NULL, "invariant");
2770
2771 // Region iteration was actually aborted. So now _finger
2772 // points to the address of the object we last scanned. If we
2773 // leave it there, when we restart this task, we will rescan
2774 // the object. It is easy to avoid this. We move the finger by
2775 // enough to point to the next possible object header (the
2776 // bitmap knows by how much we need to move it as it knows its
2777 // granularity).
2778 assert(_finger < _region_limit, "invariant");
2779 HeapWord* new_finger = _nextMarkBitMap->nextObject(_finger);
2780 // Check if bitmap iteration was aborted while scanning the last object
2781 if (new_finger >= _region_limit) {
2782 giveup_current_region();
2783 } else {
2784 move_finger_to(new_finger);
2785 }
2786 }
2787 }
2788 // At this point we have either completed iterating over the
2789 // region we were holding on to, or we have aborted.
2790
2791 // We then partially drain the local queue and the global stack.
2792 // (Do we really need this?)
2793 drain_local_queue(true);
2794 drain_global_stack(true);
2795
2796 // Read the note on the claim_region() method on why it might
2797 // return NULL with potentially more regions available for
2798 // claiming and why we have to check out_of_regions() to determine
2799 // whether we're done or not.
|
43 #include "gc/shared/gcTimer.hpp"
44 #include "gc/shared/gcTrace.hpp"
45 #include "gc/shared/gcTraceTime.inline.hpp"
46 #include "gc/shared/genOopClosures.inline.hpp"
47 #include "gc/shared/referencePolicy.hpp"
48 #include "gc/shared/strongRootsScope.hpp"
49 #include "gc/shared/taskqueue.inline.hpp"
50 #include "gc/shared/vmGCOperations.hpp"
51 #include "logging/log.hpp"
52 #include "memory/allocation.hpp"
53 #include "memory/resourceArea.hpp"
54 #include "oops/oop.inline.hpp"
55 #include "runtime/atomic.hpp"
56 #include "runtime/handles.inline.hpp"
57 #include "runtime/java.hpp"
58 #include "runtime/prefetch.inline.hpp"
59 #include "services/memTracker.hpp"
60 #include "utilities/align.hpp"
61 #include "utilities/growableArray.hpp"
62
63 void G1CMBitMap::print_on_error(outputStream* st, const char* prefix) const {
64 _bm.print_on_error(st, prefix);
65 }
66
67 size_t G1CMBitMap::compute_size(size_t heap_size) {
68 return ReservedSpace::allocation_align_size_up(heap_size / mark_distance());
69 }
70
71 size_t G1CMBitMap::mark_distance() {
72 return MinObjAlignmentInBytes * BitsPerByte;
73 }
74
75 void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) {
76 _covered = heap;
77
78 _bm = BitMapView((BitMap::bm_word_t*) storage->reserved().start(), _covered.word_size() >> _shifter);
79
80 storage->set_mapping_changed_listener(&_listener);
81 }
82
83 void G1CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) {
84 if (zero_filled) {
85 return;
86 }
87 // We need to clear the bitmap on commit, removing any existing information.
88 MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords);
89 _bm->clear_range(mr);
90 }
91
92 void G1CMBitMap::clear_range(MemRegion mr) {
93 MemRegion intersection = mr.intersection(_covered);
94 assert(!intersection.is_empty(),
95 "Given range from " PTR_FORMAT " to " PTR_FORMAT " is completely outside the heap",
96 p2i(mr.start()), p2i(mr.end()));
97 // convert address range into offset range
98 _bm.at_put_range(addr_to_offset(intersection.start()),
99 addr_to_offset(intersection.end()), false);
100 }
101
102 G1CMMarkStack::G1CMMarkStack() :
103 _max_chunk_capacity(0),
104 _base(NULL),
105 _chunk_capacity(0) {
106 set_empty();
107 }
108
109 bool G1CMMarkStack::resize(size_t new_capacity) {
110 assert(is_empty(), "Only resize when stack is empty.");
111 assert(new_capacity <= _max_chunk_capacity,
112 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
113
114 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk, mtGC>::allocate_or_null(new_capacity);
115
116 if (new_base == NULL) {
117 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
118 return false;
119 }
388 _cleanup_times(),
389 _total_counting_time(0.0),
390 _total_rs_scrub_time(0.0),
391
392 _parallel_workers(NULL),
393
394 _completed_initialization(false) {
395
396 _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage);
397 _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage);
398
399 // Create & start a ConcurrentMark thread.
400 _cmThread = new ConcurrentMarkThread(this);
401 assert(cmThread() != NULL, "CM Thread should have been created");
402 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
403 if (_cmThread->osthread() == NULL) {
404 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
405 }
406
407 assert(CGC_lock != NULL, "Where's the CGC_lock?");
408
409 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
410 satb_qs.set_buffer_size(G1SATBBufferSize);
411
412 _root_regions.init(_g1h->survivor(), this);
413
414 if (ConcGCThreads > ParallelGCThreads) {
415 log_warning(gc)("Can't have more ConcGCThreads (%u) than ParallelGCThreads (%u).",
416 ConcGCThreads, ParallelGCThreads);
417 return;
418 }
419 if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) {
420 // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent
421 // if both are set
422 _sleep_factor = 0.0;
423 _marking_task_overhead = 1.0;
424 } else if (G1MarkingOverheadPercent > 0) {
425 // We will calculate the number of parallel marking threads based
426 // on a target overhead with respect to the soft real-time goal
427 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
701 // this time no other cycle can start. So, let's make sure that this
702 // is the case.
703 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
704
705 clear_bitmap(_nextMarkBitMap, _parallel_workers, true);
706
707 // Clear the live count data. If the marking has been aborted, the abort()
708 // call already did that.
709 if (!has_aborted()) {
710 clear_live_data(_parallel_workers);
711 DEBUG_ONLY(verify_live_data_clear());
712 }
713
714 // Repeat the asserts from above.
715 guarantee(cmThread()->during_cycle(), "invariant");
716 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
717 }
718
719 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
720 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
721 clear_bitmap(_prevMarkBitMap, workers, false);
722 }
723
724 class CheckBitmapClearHRClosure : public HeapRegionClosure {
725 G1CMBitMap* _bitmap;
726 bool _error;
727 public:
728 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
729 }
730
731 virtual bool doHeapRegion(HeapRegion* r) {
732 // This closure can be called concurrently to the mutator, so we must make sure
733 // that the result of the getNextMarkedWordAddress() call is compared to the
734 // value passed to it as limit to detect any found bits.
735 // end never changes in G1.
736 HeapWord* end = r->end();
737 return _bitmap->get_next_marked_addr(r->bottom(), end) != end;
738 }
739 };
740
741 bool G1ConcurrentMark::nextMarkBitmapIsClear() {
742 CheckBitmapClearHRClosure cl(_nextMarkBitMap);
743 _g1h->heap_region_iterate(&cl);
744 return cl.complete();
745 }
746
747 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
748 public:
749 bool doHeapRegion(HeapRegion* r) {
750 r->note_start_of_marking();
751 return false;
752 }
753 };
754
755 void G1ConcurrentMark::checkpointRootsInitialPre() {
756 G1CollectedHeap* g1h = G1CollectedHeap::heap();
757
758 _has_aborted = false;
759
760 // Initialize marking structures. This has to be done in a STW phase.
761 reset();
762
763 // For each region note start of marking.
764 NoteStartOfMarkHRClosure startcl;
765 g1h->heap_region_iterate(&startcl);
766 }
767
768
769 void G1ConcurrentMark::checkpointRootsInitialPost() {
770 G1CollectedHeap* g1h = G1CollectedHeap::heap();
771
772 // Start Concurrent Marking weak-reference discovery.
773 ReferenceProcessor* rp = g1h->ref_processor_cm();
774 // enable ("weak") refs discovery
775 rp->enable_discovery();
776 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
1711 return;
1712 }
1713
1714 assert(_global_mark_stack.is_empty(), "Marking should have completed");
1715
1716 // Unload Klasses, String, Symbols, Code Cache, etc.
1717 if (ClassUnloadingWithConcurrentMark) {
1718 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm);
1719 bool purged_classes = SystemDictionary::do_unloading(&g1_is_alive, _gc_timer_cm, false /* Defer cleaning */);
1720 g1h->complete_cleaning(&g1_is_alive, purged_classes);
1721 } else {
1722 GCTraceTime(Debug, gc, phases) debug("Cleanup", _gc_timer_cm);
1723 // No need to clean string table and symbol table as they are treated as strong roots when
1724 // class unloading is disabled.
1725 g1h->partial_cleaning(&g1_is_alive, false, false, G1StringDedup::is_enabled());
1726
1727 }
1728 }
1729
1730 void G1ConcurrentMark::swapMarkBitMaps() {
1731 G1CMBitMap* temp = _prevMarkBitMap;
1732 _prevMarkBitMap = _nextMarkBitMap;
1733 _nextMarkBitMap = temp;
1734 }
1735
1736 // Closure for marking entries in SATB buffers.
1737 class G1CMSATBBufferClosure : public SATBBufferClosure {
1738 private:
1739 G1CMTask* _task;
1740 G1CollectedHeap* _g1h;
1741
1742 // This is very similar to G1CMTask::deal_with_reference, but with
1743 // more relaxed requirements for the argument, so this must be more
1744 // circumspect about treating the argument as an object.
1745 void do_entry(void* entry) const {
1746 _task->increment_refs_reached();
1747 HeapRegion* hr = _g1h->heap_region_containing(entry);
1748 if (entry < hr->next_top_at_mark_start()) {
1749 // Until we get here, we don't know whether entry refers to a valid
1750 // object; it could instead have been a stale reference.
1751 oop obj = static_cast<oop>(entry);
1752 assert(obj->is_oop(true /* ignore mark word */),
1753 "Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj));
1858 StrongRootsScope srs(active_workers);
1859
1860 G1CMRemarkTask remarkTask(this, active_workers);
1861 // We will start all available threads, even if we decide that the
1862 // active_workers will be fewer. The extra ones will just bail out
1863 // immediately.
1864 g1h->workers()->run_task(&remarkTask);
1865 }
1866
1867 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1868 guarantee(has_overflown() ||
1869 satb_mq_set.completed_buffers_num() == 0,
1870 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT,
1871 BOOL_TO_STR(has_overflown()),
1872 satb_mq_set.completed_buffers_num());
1873
1874 print_stats();
1875 }
1876
1877 void G1ConcurrentMark::clearRangePrevBitmap(MemRegion mr) {
1878 _prevMarkBitMap->clear_range(mr);
1879 }
1880
1881 HeapRegion*
1882 G1ConcurrentMark::claim_region(uint worker_id) {
1883 // "checkpoint" the finger
1884 HeapWord* finger = _finger;
1885
1886 // _heap_end will not change underneath our feet; it only changes at
1887 // yield points.
1888 while (finger < _heap_end) {
1889 assert(_g1h->is_in_g1_reserved(finger), "invariant");
1890
1891 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
1892 // Make sure that the reads below do not float before loading curr_region.
1893 OrderAccess::loadload();
1894 // Above heap_region_containing may return NULL as we always scan claim
1895 // until the end of the heap. In this case, just jump to the next region.
1896 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords;
1897
1898 // Is the gap between reading the finger and doing the CAS too long?
2105 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2106 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2107 cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());
2108 }
2109
2110 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2111 _parallel_workers->print_worker_threads_on(st);
2112 }
2113
2114 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
2115 _parallel_workers->threads_do(tc);
2116 }
2117
2118 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2119 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2120 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2121 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2122 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2123 }
2124
2125 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
2126 assert(addr < _cm->finger(), "invariant");
2127 assert(addr >= _task->finger(), "invariant");
2128
2129 // We move that task's local finger along.
2130 _task->move_finger_to(addr);
2131
2132 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
2133 // we only partially drain the local queue and global stack
2134 _task->drain_local_queue(true);
2135 _task->drain_global_stack(true);
2136
2137 // if the has_aborted flag has been raised, we need to bail out of
2138 // the iteration
2139 return !_task->has_aborted();
2140 }
2141
2142 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
2143 ReferenceProcessor* result = g1h->ref_processor_cm();
2144 assert(result != NULL, "CM reference processor should not be NULL");
2145 return result;
2146 }
2147
2148 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
2149 G1ConcurrentMark* cm,
2150 G1CMTask* task)
2151 : MetadataAwareOopClosure(get_cm_oop_closure_ref_processor(g1h)),
2152 _g1h(g1h), _cm(cm), _task(task)
2153 { }
2154
2155 void G1CMTask::setup_for_region(HeapRegion* hr) {
2156 assert(hr != NULL,
2157 "claim_region() should have filtered out NULL regions");
2158 _curr_region = hr;
2159 _finger = hr->bottom();
2160 update_region_limit();
2621
2622 double diff_prediction_ms = _g1h->g1_policy()->predictor().get_new_prediction(&_marking_step_diffs_ms);
2623 _time_target_ms = time_target_ms - diff_prediction_ms;
2624
2625 // set up the variables that are used in the work-based scheme to
2626 // call the regular clock method
2627 _words_scanned = 0;
2628 _refs_reached = 0;
2629 recalculate_limits();
2630
2631 // clear all flags
2632 clear_has_aborted();
2633 _has_timed_out = false;
2634 _draining_satb_buffers = false;
2635
2636 ++_calls;
2637
2638 // Set up the bitmap and oop closures. Anything that uses them is
2639 // eventually called from this method, so it is OK to allocate these
2640 // statically.
2641 G1CMBitMapClosure bitmap_closure(this, _cm);
2642 G1CMOopClosure cm_oop_closure(_g1h, _cm, this);
2643 set_cm_oop_closure(&cm_oop_closure);
2644
2645 if (_cm->has_overflown()) {
2646 // This can happen if the mark stack overflows during a GC pause
2647 // and this task, after a yield point, restarts. We have to abort
2648 // as we need to get into the overflow protocol which happens
2649 // right at the end of this task.
2650 set_has_aborted();
2651 }
2652
2653 // First drain any available SATB buffers. After this, we will not
2654 // look at SATB buffers before the next invocation of this method.
2655 // If enough completed SATB buffers are queued up, the regular clock
2656 // will abort this task so that it restarts.
2657 drain_satb_buffers();
2658 // ...then partially drain the local queue and the global stack
2659 drain_local_queue(true);
2660 drain_global_stack(true);
2661
2677 // through scanning this region. In this case, _finger points to
2678 // the address where we last found a marked object. If this is a
2679 // fresh region, _finger points to start().
2680 MemRegion mr = MemRegion(_finger, _region_limit);
2681
2682 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
2683 "humongous regions should go around loop once only");
2684
2685 // Some special cases:
2686 // If the memory region is empty, we can just give up the region.
2687 // If the current region is humongous then we only need to check
2688 // the bitmap for the bit associated with the start of the object,
2689 // scan the object if it's live, and give up the region.
2690 // Otherwise, let's iterate over the bitmap of the part of the region
2691 // that is left.
2692 // If the iteration is successful, give up the region.
2693 if (mr.is_empty()) {
2694 giveup_current_region();
2695 regular_clock_call();
2696 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
2697 if (_nextMarkBitMap->is_marked(mr.start())) {
2698 // The object is marked - apply the closure
2699 bitmap_closure.do_addr(mr.start());
2700 }
2701 // Even if this task aborted while scanning the humongous object
2702 // we can (and should) give up the current region.
2703 giveup_current_region();
2704 regular_clock_call();
2705 } else if (_nextMarkBitMap->iterate(&bitmap_closure, mr)) {
2706 giveup_current_region();
2707 regular_clock_call();
2708 } else {
2709 assert(has_aborted(), "currently the only way to do so");
2710 // The only way to abort the bitmap iteration is to return
2711 // false from the do_bit() method. However, inside the
2712 // do_bit() method we move the _finger to point to the
2713 // object currently being looked at. So, if we bail out, we
2714 // have definitely set _finger to something non-null.
2715 assert(_finger != NULL, "invariant");
2716
2717 // Region iteration was actually aborted. So now _finger
2718 // points to the address of the object we last scanned. If we
2719 // leave it there, when we restart this task, we will rescan
2720 // the object. It is easy to avoid this. We move the finger by
2721 // enough to point to the next possible object header.
2722 assert(_finger < _region_limit, "invariant");
2723 HeapWord* const new_finger = _finger + ((oop)_finger)->size();
2724 // Check if bitmap iteration was aborted while scanning the last object
2725 if (new_finger >= _region_limit) {
2726 giveup_current_region();
2727 } else {
2728 move_finger_to(new_finger);
2729 }
2730 }
2731 }
2732 // At this point we have either completed iterating over the
2733 // region we were holding on to, or we have aborted.
2734
2735 // We then partially drain the local queue and the global stack.
2736 // (Do we really need this?)
2737 drain_local_queue(true);
2738 drain_global_stack(true);
2739
2740 // Read the note on the claim_region() method on why it might
2741 // return NULL with potentially more regions available for
2742 // claiming and why we have to check out_of_regions() to determine
2743 // whether we're done or not.
|