845 }
846
847 // Push "obj" on the scan stack.
848 void push_on_evac_failure_scan_stack(oop obj);
849 // Process scan stack entries until the stack is empty.
850 void drain_evac_failure_scan_stack();
851 // True iff an invocation of "drain_scan_stack" is in progress; to
852 // prevent unnecessary recursion.
853 bool _drain_in_progress;
854
855 // Do any necessary initialization for evacuation-failure handling.
856 // "cl" is the closure that will be used to process evac-failure
857 // objects.
858 void init_for_evac_failure(OopsInHeapRegionClosure* cl);
859 // Do any necessary cleanup for evacuation-failure handling data
860 // structures.
861 void finalize_for_evac_failure();
862
863 // An attempt to evacuate "obj" has failed; take necessary steps.
864 oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj,
865 bool should_mark_root);
866 void handle_evacuation_failure_common(oop obj, markOop m);
867
868 // ("Weak") Reference processing support.
869 //
870 // G1 has 2 instances of the referece processor class. One
871 // (_ref_processor_cm) handles reference object discovery
872 // and subsequent processing during concurrent marking cycles.
873 //
874 // The other (_ref_processor_stw) handles reference object
875 // discovery and processing during full GCs and incremental
876 // evacuation pauses.
877 //
878 // During an incremental pause, reference discovery will be
879 // temporarily disabled for _ref_processor_cm and will be
880 // enabled for _ref_processor_stw. At the end of the evacuation
881 // pause references discovered by _ref_processor_stw will be
882 // processed and discovery will be disabled. The previous
883 // setting for reference object discovery for _ref_processor_cm
884 // will be re-instated.
885 //
1713 size_t gclab_word_size() const {
1714 return _gclab_word_size;
1715 }
1716
1717 // Calculates actual GCLab size in words
1718 size_t gclab_real_word_size() const {
1719 return bitmap_size_in_bits(pointer_delta(_real_end_word, _start_word))
1720 / BitsPerWord;
1721 }
1722
1723 static size_t bitmap_size_in_bits(size_t gclab_word_size) {
1724 size_t bits_in_bitmap = gclab_word_size >> shifter();
1725 // We are going to ensure that the beginning of a word in this
1726 // bitmap also corresponds to the beginning of a word in the
1727 // global marking bitmap. To handle the case where a GCLab
1728 // starts from the middle of the bitmap, we need to add enough
1729 // space (i.e. up to a bitmap word) to ensure that we have
1730 // enough bits in the bitmap.
1731 return bits_in_bitmap + BitsPerWord - 1;
1732 }
1733 public:
1734 GCLabBitMap(HeapWord* heap_start, size_t gclab_word_size)
1735 : BitMap(bitmap_size_in_bits(gclab_word_size)),
1736 _cm(G1CollectedHeap::heap()->concurrent_mark()),
1737 _shifter(shifter()),
1738 _bitmap_word_covers_words(bitmap_word_covers_words()),
1739 _heap_start(heap_start),
1740 _gclab_word_size(gclab_word_size),
1741 _real_start_word(NULL),
1742 _real_end_word(NULL),
1743 _start_word(NULL)
1744 {
1745 guarantee( size_in_words() >= bitmap_size_in_words(),
1746 "just making sure");
1747 }
1748
1749 inline unsigned heapWordToOffset(HeapWord* addr) {
1750 unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
1751 assert(offset < size(), "offset should be within bounds");
1752 return offset;
1802
1803 assert(start != NULL, "invariant");
1804 _real_start_word = start;
1805 _real_end_word = start + _gclab_word_size;
1806
1807 size_t diff =
1808 pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
1809 _start_word = start - diff;
1810
1811 assert(fields_well_formed(), "invariant");
1812 }
1813
1814 #ifndef PRODUCT
1815 void verify() {
1816 // verify that the marks have been propagated
1817 GCLabBitMapClosure cl(_cm, this);
1818 iterate(&cl);
1819 }
1820 #endif // PRODUCT
1821
1822 void retire() {
1823 guarantee(use_local_bitmaps, "invariant");
1824 assert(fields_well_formed(), "invariant");
1825
1826 if (_start_word != NULL) {
1827 CMBitMap* mark_bitmap = _cm->nextMarkBitMap();
1828
1829 // this means that the bitmap was set up for the GCLab
1830 assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
1831
1832 mark_bitmap->mostly_disjoint_range_union(this,
1833 0, // always start from the start of the bitmap
1834 _start_word,
1835 gclab_real_word_size());
1836 _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
1837
1838 #ifndef PRODUCT
1839 if (use_local_bitmaps && verify_local_bitmaps)
1840 verify();
1841 #endif // PRODUCT
1842 } else {
1843 assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
1844 }
1845 }
1846
1847 size_t bitmap_size_in_words() const {
1848 return (bitmap_size_in_bits(gclab_word_size()) + BitsPerWord - 1) / BitsPerWord;
1849 }
1850
1851 };
1852
1853 class G1ParGCAllocBuffer: public ParGCAllocBuffer {
1854 private:
1855 bool _retired;
1856 bool _should_mark_objects;
1857 GCLabBitMap _bitmap;
1858
1859 public:
1860 G1ParGCAllocBuffer(size_t gclab_word_size);
1861
1862 inline bool mark(HeapWord* addr) {
1863 guarantee(use_local_bitmaps, "invariant");
1864 assert(_should_mark_objects, "invariant");
1865 return _bitmap.mark(addr);
1866 }
1867
1868 inline void set_buf(HeapWord* buf) {
1869 if (use_local_bitmaps && _should_mark_objects) {
1870 _bitmap.set_buffer(buf);
1871 }
1872 ParGCAllocBuffer::set_buf(buf);
1873 _retired = false;
1874 }
1875
1876 inline void retire(bool end_of_gc, bool retain) {
1877 if (_retired)
1878 return;
1879 if (use_local_bitmaps && _should_mark_objects) {
1880 _bitmap.retire();
1881 }
1882 ParGCAllocBuffer::retire(end_of_gc, retain);
1883 _retired = true;
1884 }
1885 };
1886
1887 class G1ParScanThreadState : public StackObj {
1888 protected:
1889 G1CollectedHeap* _g1h;
1890 RefToScanQueue* _refs;
1891 DirtyCardQueue _dcq;
1892 CardTableModRefBS* _ct_bs;
1893 G1RemSet* _g1_rem;
1894
1895 G1ParGCAllocBuffer _surviving_alloc_buffer;
1896 G1ParGCAllocBuffer _tenured_alloc_buffer;
1897 G1ParGCAllocBuffer* _alloc_buffers[GCAllocPurposeCount];
1898 ageTable _age_table;
1899
1900 size_t _alloc_buffer_waste;
|
845 }
846
847 // Push "obj" on the scan stack.
848 void push_on_evac_failure_scan_stack(oop obj);
849 // Process scan stack entries until the stack is empty.
850 void drain_evac_failure_scan_stack();
851 // True iff an invocation of "drain_scan_stack" is in progress; to
852 // prevent unnecessary recursion.
853 bool _drain_in_progress;
854
855 // Do any necessary initialization for evacuation-failure handling.
856 // "cl" is the closure that will be used to process evac-failure
857 // objects.
858 void init_for_evac_failure(OopsInHeapRegionClosure* cl);
859 // Do any necessary cleanup for evacuation-failure handling data
860 // structures.
861 void finalize_for_evac_failure();
862
863 // An attempt to evacuate "obj" has failed; take necessary steps.
864 oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj,
865 bool should_mark_root,
866 int worker_i);
867 void handle_evacuation_failure_common(oop obj, markOop m);
868
869 // ("Weak") Reference processing support.
870 //
871 // G1 has 2 instances of the referece processor class. One
872 // (_ref_processor_cm) handles reference object discovery
873 // and subsequent processing during concurrent marking cycles.
874 //
875 // The other (_ref_processor_stw) handles reference object
876 // discovery and processing during full GCs and incremental
877 // evacuation pauses.
878 //
879 // During an incremental pause, reference discovery will be
880 // temporarily disabled for _ref_processor_cm and will be
881 // enabled for _ref_processor_stw. At the end of the evacuation
882 // pause references discovered by _ref_processor_stw will be
883 // processed and discovery will be disabled. The previous
884 // setting for reference object discovery for _ref_processor_cm
885 // will be re-instated.
886 //
1714 size_t gclab_word_size() const {
1715 return _gclab_word_size;
1716 }
1717
1718 // Calculates actual GCLab size in words
1719 size_t gclab_real_word_size() const {
1720 return bitmap_size_in_bits(pointer_delta(_real_end_word, _start_word))
1721 / BitsPerWord;
1722 }
1723
1724 static size_t bitmap_size_in_bits(size_t gclab_word_size) {
1725 size_t bits_in_bitmap = gclab_word_size >> shifter();
1726 // We are going to ensure that the beginning of a word in this
1727 // bitmap also corresponds to the beginning of a word in the
1728 // global marking bitmap. To handle the case where a GCLab
1729 // starts from the middle of the bitmap, we need to add enough
1730 // space (i.e. up to a bitmap word) to ensure that we have
1731 // enough bits in the bitmap.
1732 return bits_in_bitmap + BitsPerWord - 1;
1733 }
1734
1735 public:
1736 GCLabBitMap(HeapWord* heap_start, size_t gclab_word_size)
1737 : BitMap(bitmap_size_in_bits(gclab_word_size)),
1738 _cm(G1CollectedHeap::heap()->concurrent_mark()),
1739 _shifter(shifter()),
1740 _bitmap_word_covers_words(bitmap_word_covers_words()),
1741 _heap_start(heap_start),
1742 _gclab_word_size(gclab_word_size),
1743 _real_start_word(NULL),
1744 _real_end_word(NULL),
1745 _start_word(NULL)
1746 {
1747 guarantee( size_in_words() >= bitmap_size_in_words(),
1748 "just making sure");
1749 }
1750
1751 inline unsigned heapWordToOffset(HeapWord* addr) {
1752 unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
1753 assert(offset < size(), "offset should be within bounds");
1754 return offset;
1804
1805 assert(start != NULL, "invariant");
1806 _real_start_word = start;
1807 _real_end_word = start + _gclab_word_size;
1808
1809 size_t diff =
1810 pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
1811 _start_word = start - diff;
1812
1813 assert(fields_well_formed(), "invariant");
1814 }
1815
1816 #ifndef PRODUCT
1817 void verify() {
1818 // verify that the marks have been propagated
1819 GCLabBitMapClosure cl(_cm, this);
1820 iterate(&cl);
1821 }
1822 #endif // PRODUCT
1823
1824 void retire(int worker_i);
1825
1826 size_t bitmap_size_in_words() const {
1827 return (bitmap_size_in_bits(gclab_word_size()) + BitsPerWord - 1) / BitsPerWord;
1828 }
1829
1830 };
1831
1832 class G1ParGCAllocBuffer: public ParGCAllocBuffer {
1833 private:
1834 bool _retired;
1835 bool _should_mark_objects;
1836 GCLabBitMap _bitmap;
1837 int _worker_i;
1838
1839 public:
1840 G1ParGCAllocBuffer(size_t gclab_word_size, int worker_i);
1841
1842 inline bool mark(HeapWord* addr) {
1843 guarantee(use_local_bitmaps, "invariant");
1844 assert(_should_mark_objects, "invariant");
1845 return _bitmap.mark(addr);
1846 }
1847
1848 inline void set_buf(HeapWord* buf) {
1849 if (use_local_bitmaps && _should_mark_objects) {
1850 _bitmap.set_buffer(buf);
1851 }
1852 ParGCAllocBuffer::set_buf(buf);
1853 _retired = false;
1854 }
1855
1856 inline void retire(bool end_of_gc, bool retain) {
1857 if (_retired)
1858 return;
1859 if (use_local_bitmaps && _should_mark_objects) {
1860 _bitmap.retire(_worker_i);
1861 }
1862 ParGCAllocBuffer::retire(end_of_gc, retain);
1863 _retired = true;
1864 }
1865 };
1866
1867 class G1ParScanThreadState : public StackObj {
1868 protected:
1869 G1CollectedHeap* _g1h;
1870 RefToScanQueue* _refs;
1871 DirtyCardQueue _dcq;
1872 CardTableModRefBS* _ct_bs;
1873 G1RemSet* _g1_rem;
1874
1875 G1ParGCAllocBuffer _surviving_alloc_buffer;
1876 G1ParGCAllocBuffer _tenured_alloc_buffer;
1877 G1ParGCAllocBuffer* _alloc_buffers[GCAllocPurposeCount];
1878 ageTable _age_table;
1879
1880 size_t _alloc_buffer_waste;
|