826 // Returns the number of live words between bottom and TAMS.
827 size_t rebuild_rem_set_in_region(const G1CMBitMap* const bitmap,
828 HeapWord* const top_at_mark_start,
829 HeapWord* const top_at_rebuild_start,
830 HeapRegion* hr,
831 MemRegion mr) {
832 size_t marked_words = 0;
833
834 if (hr->is_humongous()) {
835 oop const humongous_obj = oop(hr->humongous_start_region()->bottom());
836 if (is_humongous_live(humongous_obj, bitmap, top_at_mark_start, top_at_rebuild_start)) {
837 // We need to scan both [bottom, TAMS) and [TAMS, top_at_rebuild_start);
838 // however in case of humongous objects it is sufficient to scan the encompassing
839 // area (top_at_rebuild_start is always larger or equal to TAMS) as one of the
840 // two areas will be zero sized. I.e. TAMS is either
841 // the same as bottom or top(_at_rebuild_start). There is no way TAMS has a different
842 // value: this would mean that TAMS points somewhere into the object.
843 assert(hr->top() == top_at_mark_start || hr->top() == top_at_rebuild_start,
844 "More than one object in the humongous region?");
845 humongous_obj->oop_iterate(&_update_cl, mr);
846 return top_at_mark_start != hr->bottom() ? mr.byte_size() : 0;
847 } else {
848 return 0;
849 }
850 }
851
852 for (LiveObjIterator it(bitmap, top_at_mark_start, mr, hr->block_start(mr.start())); it.has_next(); it.move_to_next()) {
853 oop obj = it.next();
854 size_t scanned_size = scan_for_references(obj, mr);
855 if ((HeapWord*)obj < top_at_mark_start) {
856 marked_words += scanned_size;
857 }
858 }
859
860 return marked_words * HeapWordSize;
861 }
862 public:
863 G1RebuildRemSetHeapRegionClosure(G1CollectedHeap* g1h,
864 G1ConcurrentMark* cm,
865 uint worker_id) :
866 HeapRegionClosure(),
867 _cm(cm),
868 _update_cl(g1h, worker_id) { }
869
870 bool do_heap_region(HeapRegion* hr) {
871 if (_cm->has_aborted()) {
872 return true;
873 }
874
875 uint const region_idx = hr->hrm_index();
876 DEBUG_ONLY(HeapWord* const top_at_rebuild_start_check = _cm->top_at_rebuild_start(region_idx);)
877 assert(top_at_rebuild_start_check == NULL ||
878 top_at_rebuild_start_check > hr->bottom(),
879 "A TARS (" PTR_FORMAT ") == bottom() (" PTR_FORMAT ") indicates the old region %u is empty (%s)",
880 p2i(top_at_rebuild_start_check), p2i(hr->bottom()), region_idx, hr->get_type_str());
881
882 size_t total_marked_bytes = 0;
883 size_t const chunk_size_in_words = G1RebuildRemSetChunkSize / HeapWordSize;
884
885 HeapWord* const top_at_mark_start = hr->next_top_at_mark_start();
886
887 HeapWord* cur = hr->bottom();
888 while (cur < hr->end()) {
889 // After every iteration (yield point) we need to check whether the region's
890 // TARS changed due to e.g. eager reclaim.
891 HeapWord* const top_at_rebuild_start = _cm->top_at_rebuild_start(region_idx);
892 if (top_at_rebuild_start == NULL) {
893 return false;
894 }
895
896 MemRegion next_chunk = MemRegion(hr->bottom(), top_at_rebuild_start).intersection(MemRegion(cur, chunk_size_in_words));
897 if (next_chunk.is_empty()) {
898 break;
899 }
900
901 const Ticks start = Ticks::now();
902 size_t marked_bytes = rebuild_rem_set_in_region(_cm->next_mark_bitmap(),
903 top_at_mark_start,
904 top_at_rebuild_start,
905 hr,
906 next_chunk);
907 Tickspan time = Ticks::now() - start;
908
909 log_trace(gc, remset, tracking)("Rebuilt region %u "
910 "live " SIZE_FORMAT " "
911 "time %.3fms "
912 "marked bytes " SIZE_FORMAT " "
913 "bot " PTR_FORMAT " "
914 "TAMS " PTR_FORMAT " "
915 "TARS " PTR_FORMAT,
916 region_idx,
917 _cm->liveness(region_idx) * HeapWordSize,
918 TicksToTimeHelper::seconds(time) * 1000.0,
919 marked_bytes,
920 p2i(hr->bottom()),
921 p2i(top_at_mark_start),
922 p2i(top_at_rebuild_start));
923
924 if (marked_bytes > 0) {
925 hr->add_to_marked_bytes(marked_bytes);
926 total_marked_bytes += marked_bytes;
927 }
928 cur += chunk_size_in_words;
929
930 _cm->do_yield_check();
931 if (_cm->has_aborted()) {
932 return true;
933 }
934 }
935 // In the final iteration of the loop the region might have been eagerly reclaimed.
936 // Simply filter out those regions. We can not just use region type because there
937 // might have already been new allocations into these regions.
938 DEBUG_ONLY(HeapWord* const top_at_rebuild_start = _cm->top_at_rebuild_start(region_idx);)
939 assert(!hr->is_old() ||
940 top_at_rebuild_start == NULL ||
941 total_marked_bytes == _cm->liveness(region_idx) * HeapWordSize,
942 "Marked bytes " SIZE_FORMAT " for region %u (%s) in [bottom, TAMS) do not match liveness during mark " SIZE_FORMAT " "
943 "(" PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT ")",
944 total_marked_bytes, hr->hrm_index(), hr->get_type_str(), _cm->liveness(region_idx) * HeapWordSize,
945 p2i(hr->bottom()), p2i(top_at_mark_start), p2i(top_at_rebuild_start));
946 // Abort state may have changed after the yield check.
947 return _cm->has_aborted();
948 }
949 };
950
951 HeapRegionClaimer _hr_claimer;
952 G1ConcurrentMark* _cm;
953
954 uint _worker_id_offset;
955 public:
956 G1RebuildRemSetTask(G1ConcurrentMark* cm,
957 uint n_workers,
958 uint worker_id_offset) :
959 AbstractGangTask("G1 Rebuild Remembered Set"),
960 _cm(cm),
961 _hr_claimer(n_workers),
962 _worker_id_offset(worker_id_offset) {
963 }
964
|
826 // Returns the number of live words between bottom and TAMS.
827 size_t rebuild_rem_set_in_region(const G1CMBitMap* const bitmap,
828 HeapWord* const top_at_mark_start,
829 HeapWord* const top_at_rebuild_start,
830 HeapRegion* hr,
831 MemRegion mr) {
832 size_t marked_words = 0;
833
834 if (hr->is_humongous()) {
835 oop const humongous_obj = oop(hr->humongous_start_region()->bottom());
836 if (is_humongous_live(humongous_obj, bitmap, top_at_mark_start, top_at_rebuild_start)) {
837 // We need to scan both [bottom, TAMS) and [TAMS, top_at_rebuild_start);
838 // however in case of humongous objects it is sufficient to scan the encompassing
839 // area (top_at_rebuild_start is always larger or equal to TAMS) as one of the
840 // two areas will be zero sized. I.e. TAMS is either
841 // the same as bottom or top(_at_rebuild_start). There is no way TAMS has a different
842 // value: this would mean that TAMS points somewhere into the object.
843 assert(hr->top() == top_at_mark_start || hr->top() == top_at_rebuild_start,
844 "More than one object in the humongous region?");
845 humongous_obj->oop_iterate(&_update_cl, mr);
846 return top_at_mark_start != hr->bottom() ? mr.intersection(MemRegion((HeapWord*)humongous_obj, humongous_obj->size())).byte_size() : 0;
847 } else {
848 return 0;
849 }
850 }
851
852 for (LiveObjIterator it(bitmap, top_at_mark_start, mr, hr->block_start(mr.start())); it.has_next(); it.move_to_next()) {
853 oop obj = it.next();
854 size_t scanned_size = scan_for_references(obj, mr);
855 if ((HeapWord*)obj < top_at_mark_start) {
856 marked_words += scanned_size;
857 }
858 }
859
860 return marked_words * HeapWordSize;
861 }
862 public:
863 G1RebuildRemSetHeapRegionClosure(G1CollectedHeap* g1h,
864 G1ConcurrentMark* cm,
865 uint worker_id) :
866 HeapRegionClosure(),
867 _cm(cm),
868 _update_cl(g1h, worker_id) { }
869
870 bool do_heap_region(HeapRegion* hr) {
871 if (_cm->has_aborted()) {
872 return true;
873 }
874
875 uint const region_idx = hr->hrm_index();
876 DEBUG_ONLY(HeapWord* const top_at_rebuild_start_check = _cm->top_at_rebuild_start(region_idx);)
877 assert(top_at_rebuild_start_check == NULL ||
878 top_at_rebuild_start_check > hr->bottom(),
879 "A TARS (" PTR_FORMAT ") == bottom() (" PTR_FORMAT ") indicates the old region %u is empty (%s)",
880 p2i(top_at_rebuild_start_check), p2i(hr->bottom()), region_idx, hr->get_type_str());
881
882 size_t total_marked_bytes = 0;
883 size_t const chunk_size_in_words = G1RebuildRemSetChunkSize / HeapWordSize;
884
885 HeapWord* const top_at_mark_start = hr->prev_top_at_mark_start();
886
887 HeapWord* cur = hr->bottom();
888 while (cur < hr->end()) {
889 // After every iteration (yield point) we need to check whether the region's
890 // TARS changed due to e.g. eager reclaim.
891 HeapWord* const top_at_rebuild_start = _cm->top_at_rebuild_start(region_idx);
892 if (top_at_rebuild_start == NULL) {
893 return false;
894 }
895
896 MemRegion next_chunk = MemRegion(hr->bottom(), top_at_rebuild_start).intersection(MemRegion(cur, chunk_size_in_words));
897 if (next_chunk.is_empty()) {
898 break;
899 }
900
901 const Ticks start = Ticks::now();
902 size_t marked_bytes = rebuild_rem_set_in_region(_cm->prev_mark_bitmap(),
903 top_at_mark_start,
904 top_at_rebuild_start,
905 hr,
906 next_chunk);
907 Tickspan time = Ticks::now() - start;
908
909 log_trace(gc, remset, tracking)("Rebuilt region %u "
910 "live " SIZE_FORMAT " "
911 "time %.3fms "
912 "marked bytes " SIZE_FORMAT " "
913 "bot " PTR_FORMAT " "
914 "TAMS " PTR_FORMAT " "
915 "TARS " PTR_FORMAT,
916 region_idx,
917 _cm->liveness(region_idx) * HeapWordSize,
918 TicksToTimeHelper::seconds(time) * 1000.0,
919 marked_bytes,
920 p2i(hr->bottom()),
921 p2i(top_at_mark_start),
922 p2i(top_at_rebuild_start));
923
924 if (marked_bytes > 0) {
925 total_marked_bytes += marked_bytes;
926 }
927 cur += chunk_size_in_words;
928
929 _cm->do_yield_check();
930 if (_cm->has_aborted()) {
931 return true;
932 }
933 }
934 // In the final iteration of the loop the region might have been eagerly reclaimed.
935 // Simply filter out those regions. We can not just use region type because there
936 // might have already been new allocations into these regions.
937 DEBUG_ONLY(HeapWord* const top_at_rebuild_start = _cm->top_at_rebuild_start(region_idx);)
938 assert(top_at_rebuild_start == NULL ||
939 total_marked_bytes == hr->marked_bytes(),
940 "Marked bytes " SIZE_FORMAT " for region %u (%s) in [bottom, TAMS) do not match calculated marked bytes " SIZE_FORMAT " "
941 "(" PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT ")",
942 total_marked_bytes, hr->hrm_index(), hr->get_type_str(), hr->marked_bytes(),
943 p2i(hr->bottom()), p2i(top_at_mark_start), p2i(top_at_rebuild_start));
944 // Abort state may have changed after the yield check.
945 return _cm->has_aborted();
946 }
947 };
948
949 HeapRegionClaimer _hr_claimer;
950 G1ConcurrentMark* _cm;
951
952 uint _worker_id_offset;
953 public:
954 G1RebuildRemSetTask(G1ConcurrentMark* cm,
955 uint n_workers,
956 uint worker_id_offset) :
957 AbstractGangTask("G1 Rebuild Remembered Set"),
958 _cm(cm),
959 _hr_claimer(n_workers),
960 _worker_id_offset(worker_id_offset) {
961 }
962
|