831 }
832
833 // Push "obj" on the scan stack.
834 void push_on_evac_failure_scan_stack(oop obj);
835 // Process scan stack entries until the stack is empty.
836 void drain_evac_failure_scan_stack();
837 // True iff an invocation of "drain_scan_stack" is in progress; to
838 // prevent unnecessary recursion.
839 bool _drain_in_progress;
840
841 // Do any necessary initialization for evacuation-failure handling.
842 // "cl" is the closure that will be used to process evac-failure
843 // objects.
844 void init_for_evac_failure(OopsInHeapRegionClosure* cl);
845 // Do any necessary cleanup for evacuation-failure handling data
846 // structures.
847 void finalize_for_evac_failure();
848
849 // An attempt to evacuate "obj" has failed; take necessary steps.
850 oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj,
851 bool should_mark_root);
852 void handle_evacuation_failure_common(oop obj, markOop m);
853
854 // ("Weak") Reference processing support.
855 //
856 // G1 has 2 instances of the referece processor class. One
857 // (_ref_processor_cm) handles reference object discovery
858 // and subsequent processing during concurrent marking cycles.
859 //
860 // The other (_ref_processor_stw) handles reference object
861 // discovery and processing during full GCs and incremental
862 // evacuation pauses.
863 //
864 // During an incremental pause, reference discovery will be
865 // temporarily disabled for _ref_processor_cm and will be
866 // enabled for _ref_processor_stw. At the end of the evacuation
867 // pause references discovered by _ref_processor_stw will be
868 // processed and discovery will be disabled. The previous
869 // setting for reference object discovery for _ref_processor_cm
870 // will be re-instated.
871 //
1647
1648 int _shifter;
1649 size_t _bitmap_word_covers_words;
1650
1651 // beginning of the heap
1652 HeapWord* _heap_start;
1653
1654 // this is the actual start of the GCLab
1655 HeapWord* _real_start_word;
1656
1657 // this is the actual end of the GCLab
1658 HeapWord* _real_end_word;
1659
1660 // this is the first word, possibly located before the actual start
1661 // of the GCLab, that corresponds to the first bit of the bitmap
1662 HeapWord* _start_word;
1663
1664 // size of a GCLab in words
1665 size_t _gclab_word_size;
1666
1667 static int shifter() {
1668 return MinObjAlignment - 1;
1669 }
1670
1671 // how many heap words does a single bitmap word corresponds to?
1672 static size_t bitmap_word_covers_words() {
1673 return BitsPerWord << shifter();
1674 }
1675
1676 size_t gclab_word_size() const {
1677 return _gclab_word_size;
1678 }
1679
1680 // Calculates actual GCLab size in words
1681 size_t gclab_real_word_size() const {
1682 return bitmap_size_in_bits(pointer_delta(_real_end_word, _start_word))
1683 / BitsPerWord;
1684 }
1685
1686 static size_t bitmap_size_in_bits(size_t gclab_word_size) {
1687 size_t bits_in_bitmap = gclab_word_size >> shifter();
1688 // We are going to ensure that the beginning of a word in this
1689 // bitmap also corresponds to the beginning of a word in the
1690 // global marking bitmap. To handle the case where a GCLab
1691 // starts from the middle of the bitmap, we need to add enough
1692 // space (i.e. up to a bitmap word) to ensure that we have
1693 // enough bits in the bitmap.
1694 return bits_in_bitmap + BitsPerWord - 1;
1695 }
1696 public:
1697 GCLabBitMap(HeapWord* heap_start, size_t gclab_word_size)
1698 : BitMap(bitmap_size_in_bits(gclab_word_size)),
1699 _cm(G1CollectedHeap::heap()->concurrent_mark()),
1700 _shifter(shifter()),
1701 _bitmap_word_covers_words(bitmap_word_covers_words()),
1702 _heap_start(heap_start),
1703 _gclab_word_size(gclab_word_size),
1704 _real_start_word(NULL),
1705 _real_end_word(NULL),
1706 _start_word(NULL)
1707 {
1708 guarantee( size_in_words() >= bitmap_size_in_words(),
1709 "just making sure");
1710 }
1711
1712 inline unsigned heapWordToOffset(HeapWord* addr) {
1713 unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
1714 assert(offset < size(), "offset should be within bounds");
1715 return offset;
1716 }
1717
1718 inline HeapWord* offsetToHeapWord(size_t offset) {
1719 HeapWord* addr = _start_word + (offset << _shifter);
1720 assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
1721 return addr;
1722 }
1723
1724 bool fields_well_formed() {
1725 bool ret1 = (_real_start_word == NULL) &&
1726 (_real_end_word == NULL) &&
1727 (_start_word == NULL);
1728 if (ret1)
1729 return true;
1730
1731 bool ret2 = _real_start_word >= _start_word &&
1732 _start_word < _real_end_word &&
1733 (_real_start_word + _gclab_word_size) == _real_end_word &&
1734 (_start_word + _gclab_word_size + _bitmap_word_covers_words)
1735 > _real_end_word;
1736 return ret2;
1737 }
1738
1739 inline bool mark(HeapWord* addr) {
1740 guarantee(use_local_bitmaps, "invariant");
1741 assert(fields_well_formed(), "invariant");
1742
1743 if (addr >= _real_start_word && addr < _real_end_word) {
1744 assert(!isMarked(addr), "should not have already been marked");
1745
1746 // first mark it on the bitmap
1747 at_put(heapWordToOffset(addr), true);
1748
1749 return true;
1750 } else {
1751 return false;
1752 }
1753 }
1754
1755 inline bool isMarked(HeapWord* addr) {
1756 guarantee(use_local_bitmaps, "invariant");
1757 assert(fields_well_formed(), "invariant");
1758
1759 return at(heapWordToOffset(addr));
1760 }
1761
1762 void set_buffer(HeapWord* start) {
1763 guarantee(use_local_bitmaps, "invariant");
1764 clear();
1765
1766 assert(start != NULL, "invariant");
1767 _real_start_word = start;
1768 _real_end_word = start + _gclab_word_size;
1769
1770 size_t diff =
1771 pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
1772 _start_word = start - diff;
1773
1774 assert(fields_well_formed(), "invariant");
1775 }
1776
1777 #ifndef PRODUCT
1778 void verify() {
1779 // verify that the marks have been propagated
1780 GCLabBitMapClosure cl(_cm, this);
1781 iterate(&cl);
1782 }
1783 #endif // PRODUCT
1784
1785 void retire() {
1786 guarantee(use_local_bitmaps, "invariant");
1787 assert(fields_well_formed(), "invariant");
1788
1789 if (_start_word != NULL) {
1790 CMBitMap* mark_bitmap = _cm->nextMarkBitMap();
1791
1792 // this means that the bitmap was set up for the GCLab
1793 assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
1794
1795 mark_bitmap->mostly_disjoint_range_union(this,
1796 0, // always start from the start of the bitmap
1797 _start_word,
1798 gclab_real_word_size());
1799 _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
1800
1801 #ifndef PRODUCT
1802 if (use_local_bitmaps && verify_local_bitmaps)
1803 verify();
1804 #endif // PRODUCT
1805 } else {
1806 assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
1807 }
1808 }
1809
1810 size_t bitmap_size_in_words() const {
1811 return (bitmap_size_in_bits(gclab_word_size()) + BitsPerWord - 1) / BitsPerWord;
1812 }
1813
1814 };
1815
1816 class G1ParGCAllocBuffer: public ParGCAllocBuffer {
1817 private:
1818 bool _retired;
1819 bool _should_mark_objects;
1820 GCLabBitMap _bitmap;
1821
1822 public:
1823 G1ParGCAllocBuffer(size_t gclab_word_size);
1824
1825 inline bool mark(HeapWord* addr) {
1826 guarantee(use_local_bitmaps, "invariant");
1827 assert(_should_mark_objects, "invariant");
1828 return _bitmap.mark(addr);
1829 }
1830
1831 inline void set_buf(HeapWord* buf) {
1832 if (use_local_bitmaps && _should_mark_objects) {
1833 _bitmap.set_buffer(buf);
1834 }
1835 ParGCAllocBuffer::set_buf(buf);
1836 _retired = false;
1837 }
1838
1839 inline void retire(bool end_of_gc, bool retain) {
1840 if (_retired)
1841 return;
1842 if (use_local_bitmaps && _should_mark_objects) {
1843 _bitmap.retire();
1844 }
1845 ParGCAllocBuffer::retire(end_of_gc, retain);
1846 _retired = true;
1847 }
1848 };
1849
1850 class G1ParScanThreadState : public StackObj {
1851 protected:
1852 G1CollectedHeap* _g1h;
1853 RefToScanQueue* _refs;
1854 DirtyCardQueue _dcq;
1855 CardTableModRefBS* _ct_bs;
1856 G1RemSet* _g1_rem;
1857
1858 G1ParGCAllocBuffer _surviving_alloc_buffer;
1859 G1ParGCAllocBuffer _tenured_alloc_buffer;
1860 G1ParGCAllocBuffer* _alloc_buffers[GCAllocPurposeCount];
1861 ageTable _age_table;
1862
1863 size_t _alloc_buffer_waste;
|
831 }
832
833 // Push "obj" on the scan stack.
834 void push_on_evac_failure_scan_stack(oop obj);
835 // Process scan stack entries until the stack is empty.
836 void drain_evac_failure_scan_stack();
837 // True iff an invocation of "drain_scan_stack" is in progress; to
838 // prevent unnecessary recursion.
839 bool _drain_in_progress;
840
841 // Do any necessary initialization for evacuation-failure handling.
842 // "cl" is the closure that will be used to process evac-failure
843 // objects.
844 void init_for_evac_failure(OopsInHeapRegionClosure* cl);
845 // Do any necessary cleanup for evacuation-failure handling data
846 // structures.
847 void finalize_for_evac_failure();
848
849 // An attempt to evacuate "obj" has failed; take necessary steps.
850 oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj,
851 bool should_mark_root,
852 int worker_i);
853 void handle_evacuation_failure_common(oop obj, markOop m);
854
855 // ("Weak") Reference processing support.
856 //
857 // G1 has 2 instances of the referece processor class. One
858 // (_ref_processor_cm) handles reference object discovery
859 // and subsequent processing during concurrent marking cycles.
860 //
861 // The other (_ref_processor_stw) handles reference object
862 // discovery and processing during full GCs and incremental
863 // evacuation pauses.
864 //
865 // During an incremental pause, reference discovery will be
866 // temporarily disabled for _ref_processor_cm and will be
867 // enabled for _ref_processor_stw. At the end of the evacuation
868 // pause references discovered by _ref_processor_stw will be
869 // processed and discovery will be disabled. The previous
870 // setting for reference object discovery for _ref_processor_cm
871 // will be re-instated.
872 //
1648
1649 int _shifter;
1650 size_t _bitmap_word_covers_words;
1651
1652 // beginning of the heap
1653 HeapWord* _heap_start;
1654
1655 // this is the actual start of the GCLab
1656 HeapWord* _real_start_word;
1657
1658 // this is the actual end of the GCLab
1659 HeapWord* _real_end_word;
1660
1661 // this is the first word, possibly located before the actual start
1662 // of the GCLab, that corresponds to the first bit of the bitmap
1663 HeapWord* _start_word;
1664
1665 // size of a GCLab in words
1666 size_t _gclab_word_size;
1667
1668 // The size of the marked objects in this bitmap
1669 size_t _marked_bytes;
1670
1671 // Card bitmap associated with cards spanned by the live objects.
1672 // It's sized for _gclab_word_size _plus_ 1 additional card
1673 // in case the lab does not start at a card boundary
1674 BitMap _card_bm;
1675
1676 // Card number of the base of the LAB. Used to encode
1677 // indices into the bitmap above.
1678 intptr_t _bottom_card_num;
1679
1680 static int shifter() {
1681 return MinObjAlignment - 1;
1682 }
1683
1684 // how many heap words does a single bitmap word corresponds to?
1685 static size_t bitmap_word_covers_words() {
1686 return BitsPerWord << shifter();
1687 }
1688
1689 size_t gclab_word_size() const {
1690 return _gclab_word_size;
1691 }
1692
1693 // Calculates actual GCLab size in words
1694 size_t gclab_real_word_size() const {
1695 return bitmap_size_in_bits(pointer_delta(_real_end_word, _start_word))
1696 / BitsPerWord;
1697 }
1698
1699 static size_t bitmap_size_in_bits(size_t gclab_word_size) {
1700 size_t bits_in_bitmap = gclab_word_size >> shifter();
1701 // We are going to ensure that the beginning of a word in this
1702 // bitmap also corresponds to the beginning of a word in the
1703 // global marking bitmap. To handle the case where a GCLab
1704 // starts from the middle of the bitmap, we need to add enough
1705 // space (i.e. up to a bitmap word) to ensure that we have
1706 // enough bits in the bitmap.
1707 return bits_in_bitmap + BitsPerWord - 1;
1708 }
1709
1710 public:
1711 GCLabBitMap(HeapWord* heap_start, size_t gclab_word_size)
1712 : BitMap(bitmap_size_in_bits(gclab_word_size)),
1713 _cm(G1CollectedHeap::heap()->concurrent_mark()),
1714 _shifter(shifter()),
1715 _bitmap_word_covers_words(bitmap_word_covers_words()),
1716 _heap_start(heap_start),
1717 _gclab_word_size(gclab_word_size),
1718 _real_start_word(NULL),
1719 _real_end_word(NULL),
1720 _start_word(NULL),
1721 _marked_bytes(0),
1722 _card_bm((((gclab_word_size * BytesPerWord) + CardTableModRefBS::card_size - 1) >>
1723 CardTableModRefBS::card_shift) + 1,
1724 false /* in_resource_area*/),
1725 _bottom_card_num(0)
1726 {
1727 guarantee( size_in_words() >= bitmap_size_in_words(),
1728 "just making sure");
1729 }
1730
1731 inline unsigned heapWordToOffset(HeapWord* addr) {
1732 unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
1733 assert(offset < size(), "offset should be within bounds");
1734 return offset;
1735 }
1736
1737 inline HeapWord* offsetToHeapWord(size_t offset) {
1738 HeapWord* addr = _start_word + (offset << _shifter);
1739 assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
1740 return addr;
1741 }
1742
1743 bool fields_well_formed() {
1744 bool ret1 = (_real_start_word == NULL) &&
1745 (_real_end_word == NULL) &&
1746 (_start_word == NULL);
1747 if (ret1)
1748 return true;
1749
1750 bool ret2 = _real_start_word >= _start_word &&
1751 _start_word < _real_end_word &&
1752 (_real_start_word + _gclab_word_size) == _real_end_word &&
1753 (_start_word + _gclab_word_size + _bitmap_word_covers_words)
1754 > _real_end_word;
1755 return ret2;
1756 }
1757
1758 inline bool mark(HeapWord* addr, size_t obj_size) {
1759 guarantee(use_local_bitmaps, "invariant");
1760 assert(fields_well_formed(), "invariant");
1761
1762 if (addr >= _real_start_word && addr < _real_end_word) {
1763 assert(!isMarked(addr), "should not have already been marked");
1764
1765 // first mark it on the bitmap
1766 at_put(heapWordToOffset(addr), true);
1767
1768 MemRegion mr(addr, obj_size);
1769
1770 // Add to the marked_bytes total
1771 _marked_bytes += mr.byte_size();
1772
1773 // Set the bits associated with the cards spanned by the
1774 // object
1775 intptr_t start_card_num =
1776 intptr_t(uintptr_t(mr.start()) >> CardTableModRefBS::card_shift);
1777 intptr_t last_card_num =
1778 intptr_t(uintptr_t(mr.last()) >> CardTableModRefBS::card_shift);
1779
1780 BitMap::idx_t start_idx = start_card_num - _bottom_card_num;
1781 BitMap::idx_t last_idx = last_card_num - _bottom_card_num;
1782
1783 _card_bm.set_range(start_idx, last_idx);
1784 // set_range is exclusive so set the bit for last_idx
1785 _card_bm.set_bit(last_idx);
1786
1787 return true;
1788 } else {
1789 return false;
1790 }
1791 }
1792
1793 inline bool isMarked(HeapWord* addr) {
1794 guarantee(use_local_bitmaps, "invariant");
1795 assert(fields_well_formed(), "invariant");
1796
1797 return at(heapWordToOffset(addr));
1798 }
1799
1800 void set_buffer(HeapWord* start) {
1801 guarantee(use_local_bitmaps, "invariant");
1802 clear();
1803
1804 assert(start != NULL, "invariant");
1805 _real_start_word = start;
1806 _real_end_word = start + _gclab_word_size;
1807
1808 size_t diff =
1809 pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
1810 _start_word = start - diff;
1811
1812 // Initialize _bottom_card_num for the card bitmap
1813 _bottom_card_num =
1814 intptr_t(uintptr_t(_real_start_word) >> CardTableModRefBS::card_shift);
1815
1816 _marked_bytes = 0;
1817 _card_bm.clear();
1818
1819 assert(fields_well_formed(), "invariant");
1820 }
1821
1822 #ifndef PRODUCT
1823 void verify() {
1824 // verify that the marks have been propagated
1825 GCLabBitMapClosure cl(_cm, this);
1826 iterate(&cl);
1827 }
1828 #endif // PRODUCT
1829
1830 void retire(int worker_i) {
1831 guarantee(use_local_bitmaps, "invariant");
1832 assert(fields_well_formed(), "invariant");
1833
1834 if (_start_word != NULL) {
1835 CMBitMap* mark_bitmap = _cm->nextMarkBitMap();
1836
1837 // this means that the bitmap was set up for the GCLab
1838 assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
1839
1840 mark_bitmap->mostly_disjoint_range_union(this,
1841 0, // always start from the start of the bitmap
1842 _start_word,
1843 gclab_real_word_size());
1844
1845 // Note that not all objects copied into the LAB will have a bit set
1846 // in the LAB bitmap (the LAB bitmap is used to propagate marks).
1847 // So we can't just add the entire lab and its bitmap to the count
1848 // data. The marking information has been recorded in _marked_bytes
1849 // and _card_bm.
1850 MemRegion lab_region(_real_start_word, _real_end_word);
1851 _cm->add_to_count_data_for_region(lab_region,
1852 &_card_bm,
1853 _bottom_card_num,
1854 _marked_bytes,
1855 worker_i);
1856
1857 _cm->grayRegionIfNecessary(lab_region);
1858
1859 #ifndef PRODUCT
1860 if (use_local_bitmaps && verify_local_bitmaps)
1861 verify();
1862 #endif // PRODUCT
1863 } else {
1864 assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
1865 }
1866 }
1867
1868 size_t bitmap_size_in_words() const {
1869 return (bitmap_size_in_bits(gclab_word_size()) + BitsPerWord - 1) / BitsPerWord;
1870 }
1871
1872 };
1873
1874 class G1ParGCAllocBuffer: public ParGCAllocBuffer {
1875 private:
1876 bool _retired;
1877 bool _should_mark_objects;
1878 GCLabBitMap _bitmap;
1879 int _worker_i;
1880
1881 public:
1882 G1ParGCAllocBuffer(size_t gclab_word_size, int worker_i);
1883
1884 inline bool mark(HeapWord* addr, size_t obj_size) {
1885 guarantee(use_local_bitmaps, "invariant");
1886 assert(_should_mark_objects, "invariant");
1887 return _bitmap.mark(addr, obj_size);
1888 }
1889
1890 inline void set_buf(HeapWord* buf) {
1891 if (use_local_bitmaps && _should_mark_objects) {
1892 _bitmap.set_buffer(buf);
1893 }
1894 ParGCAllocBuffer::set_buf(buf);
1895 _retired = false;
1896 }
1897
1898 inline void retire(bool end_of_gc, bool retain) {
1899 if (_retired)
1900 return;
1901 if (use_local_bitmaps && _should_mark_objects) {
1902 _bitmap.retire(_worker_i);
1903 }
1904 ParGCAllocBuffer::retire(end_of_gc, retain);
1905 _retired = true;
1906 }
1907 };
1908
1909 class G1ParScanThreadState : public StackObj {
1910 protected:
1911 G1CollectedHeap* _g1h;
1912 RefToScanQueue* _refs;
1913 DirtyCardQueue _dcq;
1914 CardTableModRefBS* _ct_bs;
1915 G1RemSet* _g1_rem;
1916
1917 G1ParGCAllocBuffer _surviving_alloc_buffer;
1918 G1ParGCAllocBuffer _tenured_alloc_buffer;
1919 G1ParGCAllocBuffer* _alloc_buffers[GCAllocPurposeCount];
1920 ageTable _age_table;
1921
1922 size_t _alloc_buffer_waste;
|