681 // false otherwise.
682 // (Rounds up to a HeapRegion boundary.)
683 bool expand(size_t expand_bytes);
684
685 // Do anything common to GC's.
686 virtual void gc_prologue(bool full);
687 virtual void gc_epilogue(bool full);
688
689 // We register a region with the fast "in collection set" test. We
690 // simply set to true the array slot corresponding to this region.
691 void register_region_with_in_cset_fast_test(HeapRegion* r) {
692 assert(_in_cset_fast_test_base != NULL, "sanity");
693 assert(r->in_collection_set(), "invariant");
694 uint index = r->hrs_index();
695 assert(index < _in_cset_fast_test_length, "invariant");
696 assert(!_in_cset_fast_test_base[index], "invariant");
697 _in_cset_fast_test_base[index] = true;
698 }
699
700 // This is a fast test on whether a reference points into the
701 // collection set or not. It does not assume that the reference
702 // points into the heap; if it doesn't, it will return false.
703 bool in_cset_fast_test(oop obj) {
704 assert(_in_cset_fast_test != NULL, "sanity");
705 if (_g1_committed.contains((HeapWord*) obj)) {
706 // no need to subtract the bottom of the heap from obj,
707 // _in_cset_fast_test is biased
708 uintx index = cast_from_oop<uintx>(obj) >> HeapRegion::LogOfHRGrainBytes;
709 bool ret = _in_cset_fast_test[index];
710 // let's make sure the result is consistent with what the slower
711 // test returns
712 assert( ret || !obj_in_cs(obj), "sanity");
713 assert(!ret || obj_in_cs(obj), "sanity");
714 return ret;
715 } else {
716 return false;
717 }
718 }
719
720 void clear_cset_fast_test() {
721 assert(_in_cset_fast_test_base != NULL, "sanity");
722 memset(_in_cset_fast_test_base, false,
723 (size_t) _in_cset_fast_test_length * sizeof(bool));
724 }
725
726 // This is called at the start of either a concurrent cycle or a Full
727 // GC to update the number of old marking cycles started.
728 void increment_old_marking_cycles_started();
729
730 // This is called at the end of either a concurrent cycle or a Full
731 // GC to update the number of old marking cycles completed. Those two
732 // can happen in a nested fashion, i.e., we start a concurrent
733 // cycle, a Full GC happens half-way through it which ends first,
734 // and then the cycle notices that a Full GC happened and ends
735 // too. The concurrent parameter is a boolean to help us do a bit
736 // tighter consistency checking in the method. If concurrent is
737 // false, the caller is the inner caller in the nesting (i.e., the
1864 _priority_buffer[last] = retired_and_set;
1865 }
1866 };
1867
1868 class G1ParScanThreadState : public StackObj {
1869 protected:
1870 G1CollectedHeap* _g1h;
1871 RefToScanQueue* _refs;
1872 DirtyCardQueue _dcq;
1873 G1SATBCardTableModRefBS* _ct_bs;
1874 G1RemSet* _g1_rem;
1875
1876 G1ParGCAllocBufferContainer _surviving_alloc_buffer;
1877 G1ParGCAllocBufferContainer _tenured_alloc_buffer;
1878 G1ParGCAllocBufferContainer* _alloc_buffers[GCAllocPurposeCount];
1879 ageTable _age_table;
1880
1881 size_t _alloc_buffer_waste;
1882 size_t _undo_waste;
1883
1884 OopsInHeapRegionClosure* _evac_failure_cl;
1885 G1ParScanHeapEvacClosure* _evac_cl;
1886 G1ParScanPartialArrayClosure* _partial_scan_cl;
1887
1888 int _hash_seed;
1889 uint _queue_num;
1890
1891 size_t _term_attempts;
1892
1893 double _start;
1894 double _start_strong_roots;
1895 double _strong_roots_time;
1896 double _start_term;
1897 double _term_time;
1898
1899 // Map from young-age-index (0 == not young, 1 is youngest) to
1900 // surviving words. base is what we get back from the malloc call
1901 size_t* _surviving_young_words_base;
1902 // this points into the array, as we use the first few entries for padding
1903 size_t* _surviving_young_words;
1904
1905 #define PADDING_ELEM_NUM (DEFAULT_CACHE_LINE_SIZE / sizeof(size_t))
1906
1913
1914 template <class T> void immediate_rs_update(HeapRegion* from, T* p, int tid) {
1915 if (!from->is_survivor()) {
1916 _g1_rem->par_write_ref(from, p, tid);
1917 }
1918 }
1919
1920 template <class T> void deferred_rs_update(HeapRegion* from, T* p, int tid) {
1921 // If the new value of the field points to the same region or
1922 // is the to-space, we don't need to include it in the Rset updates.
1923 if (!from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && !from->is_survivor()) {
1924 size_t card_index = ctbs()->index_for(p);
1925 // If the card hasn't been added to the buffer, do it.
1926 if (ctbs()->mark_card_deferred(card_index)) {
1927 dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
1928 }
1929 }
1930 }
1931
1932 public:
1933 G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num);
1934
1935 ~G1ParScanThreadState() {
1936 FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base, mtGC);
1937 }
1938
1939 RefToScanQueue* refs() { return _refs; }
1940 ageTable* age_table() { return &_age_table; }
1941
1942 G1ParGCAllocBufferContainer* alloc_buffer(GCAllocPurpose purpose) {
1943 return _alloc_buffers[purpose];
1944 }
1945
1946 size_t alloc_buffer_waste() const { return _alloc_buffer_waste; }
1947 size_t undo_waste() const { return _undo_waste; }
1948
1949 #ifdef ASSERT
1950 bool verify_ref(narrowOop* ref) const;
1951 bool verify_ref(oop* ref) const;
1952 bool verify_task(StarTask ref) const;
1953 #endif // ASSERT
1992 }
1993
1994 void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
1995 if (alloc_buffer(purpose)->contains(obj)) {
1996 assert(alloc_buffer(purpose)->contains(obj + word_sz - 1),
1997 "should contain whole object");
1998 alloc_buffer(purpose)->undo_allocation(obj, word_sz);
1999 } else {
2000 CollectedHeap::fill_with_object(obj, word_sz);
2001 add_to_undo_waste(word_sz);
2002 }
2003 }
2004
2005 void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
2006 _evac_failure_cl = evac_failure_cl;
2007 }
2008 OopsInHeapRegionClosure* evac_failure_closure() {
2009 return _evac_failure_cl;
2010 }
2011
2012 void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) {
2013 _evac_cl = evac_cl;
2014 }
2015
2016 void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) {
2017 _partial_scan_cl = partial_scan_cl;
2018 }
2019
2020 int* hash_seed() { return &_hash_seed; }
2021 uint queue_num() { return _queue_num; }
2022
2023 size_t term_attempts() const { return _term_attempts; }
2024 void note_term_attempt() { _term_attempts++; }
2025
2026 void start_strong_roots() {
2027 _start_strong_roots = os::elapsedTime();
2028 }
2029 void end_strong_roots() {
2030 _strong_roots_time += (os::elapsedTime() - _start_strong_roots);
2031 }
2032 double strong_roots_time() const { return _strong_roots_time; }
2033
2034 void start_term_time() {
2035 note_term_attempt();
2036 _start_term = os::elapsedTime();
2037 }
2038 void end_term_time() {
2039 _term_time += (os::elapsedTime() - _start_term);
2048 print_termination_stats_hdr(outputStream* const st = gclog_or_tty);
2049 void
2050 print_termination_stats(int i, outputStream* const st = gclog_or_tty) const;
2051
2052 size_t* surviving_young_words() {
2053 // We add on to hide entry 0 which accumulates surviving words for
2054 // age -1 regions (i.e. non-young ones)
2055 return _surviving_young_words;
2056 }
2057
2058 void retire_alloc_buffers() {
2059 for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
2060 size_t waste = _alloc_buffers[ap]->words_remaining();
2061 add_to_alloc_buffer_waste(waste);
2062 _alloc_buffers[ap]->flush_stats_and_retire(_g1h->stats_for_purpose((GCAllocPurpose)ap),
2063 true /* end_of_gc */,
2064 false /* retain */);
2065 }
2066 }
2067
2068 template <class T> void deal_with_reference(T* ref_to_scan) {
2069 if (has_partial_array_mask(ref_to_scan)) {
2070 _partial_scan_cl->do_oop_nv(ref_to_scan);
2071 } else {
2072 // Note: we can use "raw" versions of "region_containing" because
2073 // "obj_to_scan" is definitely in the heap, and is not in a
2074 // humongous region.
2075 HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
2076 _evac_cl->set_region(r);
2077 _evac_cl->do_oop_nv(ref_to_scan);
2078 }
2079 }
2080
2081 void deal_with_reference(StarTask ref) {
2082 assert(verify_task(ref), "sanity");
2083 if (ref.is_narrow()) {
2084 deal_with_reference((narrowOop*)ref);
2085 } else {
2086 deal_with_reference((oop*)ref);
2087 }
2088 }
2089
2090 void trim_queue();
2091 };
2092
2093 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_HPP
|
681 // false otherwise.
682 // (Rounds up to a HeapRegion boundary.)
683 bool expand(size_t expand_bytes);
684
685 // Do anything common to GC's.
686 virtual void gc_prologue(bool full);
687 virtual void gc_epilogue(bool full);
688
689 // We register a region with the fast "in collection set" test. We
690 // simply set to true the array slot corresponding to this region.
691 void register_region_with_in_cset_fast_test(HeapRegion* r) {
692 assert(_in_cset_fast_test_base != NULL, "sanity");
693 assert(r->in_collection_set(), "invariant");
694 uint index = r->hrs_index();
695 assert(index < _in_cset_fast_test_length, "invariant");
696 assert(!_in_cset_fast_test_base[index], "invariant");
697 _in_cset_fast_test_base[index] = true;
698 }
699
700 // This is a fast test on whether a reference points into the
701 // collection set or not. Assume that the reference
702 // points into the heap.
703 bool in_cset_fast_test(oop obj) {
704 assert(_in_cset_fast_test != NULL, "sanity");
705 assert(_g1_committed.contains((HeapWord*) obj), "invariant");
706 // no need to subtract the bottom of the heap from obj,
707 // _in_cset_fast_test is biased
708 uintx index = cast_from_oop<uintx>(obj) >> HeapRegion::LogOfHRGrainBytes;
709 bool ret = _in_cset_fast_test[index];
710 // let's make sure the result is consistent with what the slower
711 // test returns
712 assert( ret || !obj_in_cs(obj), "sanity");
713 assert(!ret || obj_in_cs(obj), "sanity");
714 return ret;
715 }
716
717 void clear_cset_fast_test() {
718 assert(_in_cset_fast_test_base != NULL, "sanity");
719 memset(_in_cset_fast_test_base, false,
720 (size_t) _in_cset_fast_test_length * sizeof(bool));
721 }
722
723 // This is called at the start of either a concurrent cycle or a Full
724 // GC to update the number of old marking cycles started.
725 void increment_old_marking_cycles_started();
726
727 // This is called at the end of either a concurrent cycle or a Full
728 // GC to update the number of old marking cycles completed. Those two
729 // can happen in a nested fashion, i.e., we start a concurrent
730 // cycle, a Full GC happens half-way through it which ends first,
731 // and then the cycle notices that a Full GC happened and ends
732 // too. The concurrent parameter is a boolean to help us do a bit
733 // tighter consistency checking in the method. If concurrent is
734 // false, the caller is the inner caller in the nesting (i.e., the
1861 _priority_buffer[last] = retired_and_set;
1862 }
1863 };
1864
1865 class G1ParScanThreadState : public StackObj {
1866 protected:
1867 G1CollectedHeap* _g1h;
1868 RefToScanQueue* _refs;
1869 DirtyCardQueue _dcq;
1870 G1SATBCardTableModRefBS* _ct_bs;
1871 G1RemSet* _g1_rem;
1872
1873 G1ParGCAllocBufferContainer _surviving_alloc_buffer;
1874 G1ParGCAllocBufferContainer _tenured_alloc_buffer;
1875 G1ParGCAllocBufferContainer* _alloc_buffers[GCAllocPurposeCount];
1876 ageTable _age_table;
1877
1878 size_t _alloc_buffer_waste;
1879 size_t _undo_waste;
1880
1881 G1ParScanClosure _scanner;
1882 OopsInHeapRegionClosure* _evac_failure_cl;
1883
1884 int _hash_seed;
1885 uint _queue_num;
1886
1887 size_t _term_attempts;
1888
1889 double _start;
1890 double _start_strong_roots;
1891 double _strong_roots_time;
1892 double _start_term;
1893 double _term_time;
1894
1895 // Map from young-age-index (0 == not young, 1 is youngest) to
1896 // surviving words. base is what we get back from the malloc call
1897 size_t* _surviving_young_words_base;
1898 // this points into the array, as we use the first few entries for padding
1899 size_t* _surviving_young_words;
1900
1901 #define PADDING_ELEM_NUM (DEFAULT_CACHE_LINE_SIZE / sizeof(size_t))
1902
1909
1910 template <class T> void immediate_rs_update(HeapRegion* from, T* p, int tid) {
1911 if (!from->is_survivor()) {
1912 _g1_rem->par_write_ref(from, p, tid);
1913 }
1914 }
1915
1916 template <class T> void deferred_rs_update(HeapRegion* from, T* p, int tid) {
1917 // If the new value of the field points to the same region or
1918 // is the to-space, we don't need to include it in the Rset updates.
1919 if (!from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && !from->is_survivor()) {
1920 size_t card_index = ctbs()->index_for(p);
1921 // If the card hasn't been added to the buffer, do it.
1922 if (ctbs()->mark_card_deferred(card_index)) {
1923 dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
1924 }
1925 }
1926 }
1927
1928 public:
1929 G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num, ReferenceProcessor* rp);
1930
1931 ~G1ParScanThreadState() {
1932 FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base, mtGC);
1933 }
1934
1935 RefToScanQueue* refs() { return _refs; }
1936 ageTable* age_table() { return &_age_table; }
1937
1938 G1ParGCAllocBufferContainer* alloc_buffer(GCAllocPurpose purpose) {
1939 return _alloc_buffers[purpose];
1940 }
1941
1942 size_t alloc_buffer_waste() const { return _alloc_buffer_waste; }
1943 size_t undo_waste() const { return _undo_waste; }
1944
1945 #ifdef ASSERT
1946 bool verify_ref(narrowOop* ref) const;
1947 bool verify_ref(oop* ref) const;
1948 bool verify_task(StarTask ref) const;
1949 #endif // ASSERT
1988 }
1989
1990 void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
1991 if (alloc_buffer(purpose)->contains(obj)) {
1992 assert(alloc_buffer(purpose)->contains(obj + word_sz - 1),
1993 "should contain whole object");
1994 alloc_buffer(purpose)->undo_allocation(obj, word_sz);
1995 } else {
1996 CollectedHeap::fill_with_object(obj, word_sz);
1997 add_to_undo_waste(word_sz);
1998 }
1999 }
2000
2001 void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
2002 _evac_failure_cl = evac_failure_cl;
2003 }
2004 OopsInHeapRegionClosure* evac_failure_closure() {
2005 return _evac_failure_cl;
2006 }
2007
2008 int* hash_seed() { return &_hash_seed; }
2009 uint queue_num() { return _queue_num; }
2010
2011 size_t term_attempts() const { return _term_attempts; }
2012 void note_term_attempt() { _term_attempts++; }
2013
2014 void start_strong_roots() {
2015 _start_strong_roots = os::elapsedTime();
2016 }
2017 void end_strong_roots() {
2018 _strong_roots_time += (os::elapsedTime() - _start_strong_roots);
2019 }
2020 double strong_roots_time() const { return _strong_roots_time; }
2021
2022 void start_term_time() {
2023 note_term_attempt();
2024 _start_term = os::elapsedTime();
2025 }
2026 void end_term_time() {
2027 _term_time += (os::elapsedTime() - _start_term);
2036 print_termination_stats_hdr(outputStream* const st = gclog_or_tty);
2037 void
2038 print_termination_stats(int i, outputStream* const st = gclog_or_tty) const;
2039
2040 size_t* surviving_young_words() {
2041 // We add on to hide entry 0 which accumulates surviving words for
2042 // age -1 regions (i.e. non-young ones)
2043 return _surviving_young_words;
2044 }
2045
2046 void retire_alloc_buffers() {
2047 for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
2048 size_t waste = _alloc_buffers[ap]->words_remaining();
2049 add_to_alloc_buffer_waste(waste);
2050 _alloc_buffers[ap]->flush_stats_and_retire(_g1h->stats_for_purpose((GCAllocPurpose)ap),
2051 true /* end_of_gc */,
2052 false /* retain */);
2053 }
2054 }
2055
2056 private:
2057 #define G1_PARTIAL_ARRAY_MASK 0x2
2058
2059 inline bool has_partial_array_mask(oop* const ref) const {
2060 return ((uintptr_t)ref & G1_PARTIAL_ARRAY_MASK) == G1_PARTIAL_ARRAY_MASK;
2061 }
2062
2063 // We never encode partial array oops as narrowOop*, so return false immediately.
2064 // This allows the compiler to create optimized code when popping references from
2065 // the work queue.
2066 inline bool has_partial_array_mask(narrowOop* const ref) const {
2067 assert(((uintptr_t)ref & G1_PARTIAL_ARRAY_MASK) != G1_PARTIAL_ARRAY_MASK, "Partial array oop reference encoded as narrowOop*");
2068 return false;
2069 }
2070
2071 // Only implement set_partial_array_mask() for regular oops, not for narrowOops.
2072 // We always encode partial arrays as regular oop, to allow the
2073 // specialization for has_partial_array_mask() for narrowOops above.
2074 // This means that unintentional use of this method with narrowOops are caught
2075 // by the compiler.
2076 inline oop* set_partial_array_mask(oop obj) {
2077 assert(((uintptr_t)(void *)obj & G1_PARTIAL_ARRAY_MASK) == 0, "Information loss!");
2078 return (oop*) ((uintptr_t)(void *)obj | G1_PARTIAL_ARRAY_MASK);
2079 }
2080
2081 // We always encode continuations as oop*, so we only need clear_partial_array_mask()
2082 // with an oop* parameter.
2083 inline oop clear_partial_array_mask(oop* ref) const {
2084 return cast_to_oop((intptr_t)ref & ~G1_PARTIAL_ARRAY_MASK);
2085 }
2086
2087 static int min_array_chunking_size() {
2088 return 2 * ParGCArrayScanChunk;
2089 }
2090
2091 static bool obj_needs_chunking(oop obj, size_t word_size) {
2092 return (word_size > (size_t)min_array_chunking_size()) && obj->is_objArray();
2093 }
2094
2095 void do_oop_partial_array(oop* p) {
2096 assert(has_partial_array_mask(p), "invariant");
2097 oop from_obj = clear_partial_array_mask(p);
2098
2099 assert(Universe::heap()->is_in_reserved(from_obj), "must be in heap.");
2100 assert(from_obj->is_objArray(), "must be obj array");
2101 objArrayOop from_obj_array = objArrayOop(from_obj);
2102 // The from-space object contains the real length.
2103 int length = from_obj_array->length();
2104
2105 assert(from_obj->is_forwarded(), "must be forwarded");
2106 oop to_obj = from_obj->forwardee();
2107 assert(from_obj != to_obj, "should not be chunking self-forwarded objects");
2108 objArrayOop to_obj_array = objArrayOop(to_obj);
2109 // We keep track of the next start index in the length field of the
2110 // to-space object.
2111 int next_index = to_obj_array->length();
2112 assert(0 <= next_index && next_index < length,
2113 err_msg("invariant, next index: %d, length: %d", next_index, length));
2114
2115 int start = next_index;
2116 int end = length;
2117 assert(start <= end, "invariant");
2118 int remainder = end - start;
2119 // We'll try not to push a range that's smaller than ParGCArrayScanChunk.
2120 if (remainder > min_array_chunking_size()) {
2121 end = start + ParGCArrayScanChunk;
2122 to_obj_array->set_length(end);
2123 // Push the remainder before we process the range in case another
2124 // worker has run out of things to do and can steal it.
2125 oop* from_obj_p = set_partial_array_mask(from_obj);
2126 push_on_queue(from_obj_p);
2127 } else {
2128 assert(length == end, "sanity");
2129 // We'll process the final range for this object. Restore the length
2130 // so that the heap remains parsable in case of evacuation failure.
2131 to_obj_array->set_length(end);
2132 }
2133 _scanner.set_region(_g1h->heap_region_containing_raw(to_obj));
2134 // Process indexes [start,end). It will also process the header
2135 // along with the first chunk (i.e., the chunk with start == 0).
2136 // Note that at this point the length field of to_obj_array is not
2137 // correct given that we are using it to keep track of the next
2138 // start index. oop_iterate_range() (thankfully!) ignores the length
2139 // field and only relies on the start / end parameters. It does
2140 // however return the size of the object which will be incorrect. So
2141 // we have to ignore it even if we wanted to use it.
2142 to_obj_array->oop_iterate_range(&_scanner, start, end);
2143 }
2144
2145 // This method is applied to the fields of the objects that have just been copied.
2146 template <class T> void do_oop_evac(T* p, HeapRegion* from) {
2147 assert(!oopDesc::is_null(oopDesc::load_decode_heap_oop(p)),
2148 "Reference should not be NULL here as such are never pushed to the task queue.");
2149 oop obj = oopDesc::load_decode_heap_oop_not_null(p);
2150
2151 // Although we never intentionally push references outside of the collection
2152 // set, due to (benign) races in the claim mechanism during RSet scanning more
2153 // than one thread might claim the same card. So the same card may be
2154 // processed multiple times. So redo this check.
2155 if (_g1h->in_cset_fast_test(obj)) {
2156 oop forwardee;
2157 if (obj->is_forwarded()) {
2158 forwardee = obj->forwardee();
2159 } else {
2160 forwardee = copy_to_survivor_space(obj);
2161 }
2162 assert(forwardee != NULL, "forwardee should not be NULL");
2163 oopDesc::encode_store_heap_oop(p, forwardee);
2164 }
2165
2166 assert(obj != NULL, "Must be");
2167 update_rs(from, p, queue_num());
2168 }
2169
2170 public:
2171
2172 oop copy_to_survivor_space(oop old);
2173
2174 template <class T> void deal_with_reference(T* ref_to_scan) {
2175 if (!has_partial_array_mask(ref_to_scan)) {
2176 // Note: we can use "raw" versions of "region_containing" because
2177 // "obj_to_scan" is definitely in the heap, and is not in a
2178 // humongous region.
2179 HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
2180 do_oop_evac(ref_to_scan, r);
2181 } else {
2182 // Partial arrays are always encoded as oop*. Cast here to avoid generating
2183 // the superfluous additional method.
2184 do_oop_partial_array((oop*)ref_to_scan);
2185 }
2186 }
2187
2188 void deal_with_reference(StarTask ref) {
2189 assert(verify_task(ref), "sanity");
2190 if (ref.is_narrow()) {
2191 deal_with_reference((narrowOop*)ref);
2192 } else {
2193 deal_with_reference((oop*)ref);
2194 }
2195 }
2196
2197 void trim_queue();
2198 };
2199
2200 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_HPP
|