8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_GC_SERIAL_MARKSWEEP_HPP
26 #define SHARE_VM_GC_SERIAL_MARKSWEEP_HPP
27
28 #include "gc/shared/collectedHeap.hpp"
29 #include "gc/shared/genOopClosures.hpp"
30 #include "gc/shared/taskqueue.hpp"
31 #include "memory/iterator.hpp"
32 #include "oops/markOop.hpp"
33 #include "oops/oop.hpp"
34 #include "runtime/timer.hpp"
35 #include "utilities/growableArray.hpp"
36 #include "utilities/stack.hpp"
37
38 class ReferenceProcessor;
39 class DataLayout;
40 class SerialOldTracer;
41 class STWGCTimer;
42
43 // MarkSweep takes care of global mark-compact garbage collection for a
44 // GenCollectedHeap using a four-phase pointer forwarding algorithm. All
45 // generations are assumed to support marking; those that can also support
46 // compaction.
47 //
48 // Class unloading will only occur when a full gc is invoked.
49
50 // declared at end
51 class PreservedMark;
52
53 class MarkSweep : AllStatic {
54 //
55 // Inline closure decls
56 //
57 class FollowRootClosure: public OopsInGenClosure {
58 public:
59 virtual void do_oop(oop* p);
60 virtual void do_oop(narrowOop* p);
61 };
62
63 class MarkAndPushClosure: public ExtendedOopClosure {
64 public:
65 template <typename T> void do_oop_nv(T* p);
66 virtual void do_oop(oop* p);
67 virtual void do_oop(narrowOop* p);
68 };
69
70 class FollowStackClosure: public VoidClosure {
71 public:
85 // Used for java/lang/ref handling
86 class IsAliveClosure: public BoolObjectClosure {
87 public:
88 virtual bool do_object_b(oop p);
89 };
90
91 class KeepAliveClosure: public OopClosure {
92 protected:
93 template <class T> void do_oop_work(T* p);
94 public:
95 virtual void do_oop(oop* p);
96 virtual void do_oop(narrowOop* p);
97 };
98
99 //
100 // Friend decls
101 //
102 friend class AdjustPointerClosure;
103 friend class KeepAliveClosure;
104 friend class VM_MarkSweep;
105 friend void marksweep_init();
106
107 //
108 // Vars
109 //
110 protected:
111 // Total invocations of a MarkSweep collection
112 static uint _total_invocations;
113
114 // Traversal stacks used during phase1
115 static Stack<oop, mtGC> _marking_stack;
116 static Stack<ObjArrayTask, mtGC> _objarray_stack;
117
118 // Space for storing/restoring mark word
119 static Stack<markOop, mtGC> _preserved_mark_stack;
120 static Stack<oop, mtGC> _preserved_oop_stack;
121 static size_t _preserved_count;
122 static size_t _preserved_count_max;
123 static PreservedMark* _preserved_marks;
124
125 // Reference processing (used in ...follow_contents)
126 static ReferenceProcessor* _ref_processor;
127
128 static STWGCTimer* _gc_timer;
129 static SerialOldTracer* _gc_tracer;
130
131 // Non public closures
132 static KeepAliveClosure keep_alive;
133
134 public:
135 // Public closures
136 static IsAliveClosure is_alive;
137 static FollowRootClosure follow_root_closure;
138 static MarkAndPushClosure mark_and_push_closure;
139 static FollowStackClosure follow_stack_closure;
140 static CLDToOopClosure follow_cld_closure;
141 static AdjustPointerClosure adjust_pointer_closure;
142 static CLDToOopClosure adjust_cld_closure;
143
144 // Accessors
145 static uint total_invocations() { return _total_invocations; }
146
147 // Reference Processing
148 static ReferenceProcessor* const ref_processor() { return _ref_processor; }
149
150 // Archive Object handling
151 static inline bool is_archive_object(oop object);
152
153 static STWGCTimer* gc_timer() { return _gc_timer; }
154 static SerialOldTracer* gc_tracer() { return _gc_tracer; }
155
156 // Call backs for marking
157 static void mark_object(oop obj);
158 // Mark pointer and follow contents. Empty marking stack afterwards.
159 template <class T> static inline void follow_root(T* p);
160
161 // Check mark and maybe push on marking stack
162 template <class T> static void mark_and_push(T* p);
163
164 static inline void push_objarray(oop obj, size_t index);
165
166 static void follow_stack(); // Empty marking stack.
167
168 static void follow_object(oop obj);
169
170 static void follow_array(objArrayOop array, int index);
171
172 static void follow_klass(Klass* klass);
173
174 static void follow_class_loader(ClassLoaderData* cld);
175
176 static int adjust_pointers(oop obj);
177
178 static void preserve_mark(oop p, markOop mark);
179 // Save the mark word so it can be restored later
180 static void adjust_marks(); // Adjust the pointers in the preserved marks table
181 static void restore_marks(); // Restore the marks that we saved in preserve_mark
182
183 template <class T> static inline void adjust_pointer(T* p);
184 };
185
186 class PreservedMark VALUE_OBJ_CLASS_SPEC {
187 private:
188 oop _obj;
189 markOop _mark;
190
191 public:
192 void init(oop obj, markOop mark) {
193 _obj = obj;
194 _mark = mark;
195 }
196
197 void adjust_pointer();
198 void restore();
199 };
200
201 #endif // SHARE_VM_GC_SERIAL_MARKSWEEP_HPP
|
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_GC_SERIAL_MARKSWEEP_HPP
26 #define SHARE_VM_GC_SERIAL_MARKSWEEP_HPP
27
28 #include "gc/cms/compactibleFreeListSpace.hpp"
29 #include "gc/shared/collectedHeap.hpp"
30 #include "gc/shared/genCollectedHeap.hpp"
31 #include "gc/shared/genOopClosures.hpp"
32 #include "gc/shared/space.hpp"
33 #include "gc/shared/strongRootsScope.hpp"
34 #include "gc/shared/taskqueue.inline.hpp"
35 #include "memory/iterator.hpp"
36 #include "oops/markOop.hpp"
37 #include "oops/oop.hpp"
38 #include "runtime/prefetch.inline.hpp"
39 #include "runtime/timer.hpp"
40 #include "utilities/growableArray.hpp"
41 #include "utilities/stack.hpp"
42
43 class ReferenceProcessor;
44 class DataLayout;
45 class SerialOldTracer;
46 class STWGCTimer;
47
48 // MarkSweep takes care of global mark-compact garbage collection for a
49 // GenCollectedHeap using a four-phase pointer forwarding algorithm. All
50 // generations are assumed to support marking; those that can also support
51 // compaction.
52 //
53 // Class unloading will only occur when a full gc is invoked.
54
55 // declared at end
56 class PreservedMark;
57 class PMSMarkBitMap;
58 class PMSRegion;
59 class PMSRegionArray;
60 class PMSRegionArraySet;
61 typedef OverflowTaskQueue<PMSRegion*, mtGC> PMSRegionTaskQueue;
62 typedef GenericTaskQueueSet<PMSRegionTaskQueue, mtGC> PMSRegionTaskQueueSet;
63
64 class MarkSweep : AllStatic {
65 //
66 // Inline closure decls
67 //
68 class FollowRootClosure: public OopsInGenClosure {
69 public:
70 virtual void do_oop(oop* p);
71 virtual void do_oop(narrowOop* p);
72 };
73
74 class MarkAndPushClosure: public ExtendedOopClosure {
75 public:
76 template <typename T> void do_oop_nv(T* p);
77 virtual void do_oop(oop* p);
78 virtual void do_oop(narrowOop* p);
79 };
80
81 class FollowStackClosure: public VoidClosure {
82 public:
96 // Used for java/lang/ref handling
97 class IsAliveClosure: public BoolObjectClosure {
98 public:
99 virtual bool do_object_b(oop p);
100 };
101
102 class KeepAliveClosure: public OopClosure {
103 protected:
104 template <class T> void do_oop_work(T* p);
105 public:
106 virtual void do_oop(oop* p);
107 virtual void do_oop(narrowOop* p);
108 };
109
110 //
111 // Friend decls
112 //
113 friend class AdjustPointerClosure;
114 friend class KeepAliveClosure;
115 friend class VM_MarkSweep;
116 friend class PMSRefProcTask;
117 friend void marksweep_init();
118
119 public:
120 static void initialize_parallel_mark_sweep();
121 //
122 // Vars
123 //
124 protected:
125 // Total invocations of a MarkSweep collection
126 static uint _total_invocations;
127
128 // Traversal stacks used during phase1
129 static Stack<oop, mtGC> _marking_stack;
130 static Stack<ObjArrayTask, mtGC> _objarray_stack;
131
132 // Space for storing/restoring mark word
133 static Stack<markOop, mtGC> _preserved_mark_stack;
134 static Stack<oop, mtGC> _preserved_oop_stack;
135 static size_t _preserved_count;
136 static size_t _preserved_count_max;
137 static PreservedMark* _preserved_marks;
138
139 // For parallel mark sweep (PMS)
140 static PMSMarkBitMap* _pms_mark_bit_map;
141 static PMSRegionArraySet* _pms_region_array_set;
142 static volatile intptr_t _pms_mark_counter;
143 static PMSRegionTaskQueueSet* _pms_region_task_queues;
144
145 // Reference processing (used in ...follow_contents)
146 static ReferenceProcessor* _ref_processor;
147
148 static STWGCTimer* _gc_timer;
149 static SerialOldTracer* _gc_tracer;
150
151 // Non public closures
152 static KeepAliveClosure keep_alive;
153
154 public:
155 // Public closures
156 static IsAliveClosure is_alive;
157 static FollowRootClosure follow_root_closure;
158 static MarkAndPushClosure mark_and_push_closure;
159 static FollowStackClosure follow_stack_closure;
160 static CLDToOopClosure follow_cld_closure;
161 static AdjustPointerClosure adjust_pointer_closure;
162 static CLDToOopClosure adjust_cld_closure;
163
164 // Accessors
165 static uint total_invocations() { return _total_invocations; }
166
167 // Reference Processing
168 static ReferenceProcessor* const ref_processor() { return _ref_processor; }
169
170 // Archive Object handling
171 static inline bool is_archive_object(oop object);
172
173 static STWGCTimer* gc_timer() { return _gc_timer; }
174 static SerialOldTracer* gc_tracer() { return _gc_tracer; }
175
176 static PMSMarkBitMap* pms_mark_bit_map() { return _pms_mark_bit_map; }
177 static PMSRegionArraySet* pms_region_array_set() { return _pms_region_array_set; }
178 static PMSRegionTaskQueueSet* pms_region_task_queues() { return _pms_region_task_queues; }
179
180
181 // Call backs for marking
182 static void mark_object(oop obj);
183 static bool par_mark_object(oop obj);
184 static bool is_object_marked(oop obj);
185 // Mark pointer and follow contents. Empty marking stack afterwards.
186 template <class T> static inline void follow_root(T* p);
187
188 // Check mark and maybe push on marking stack
189 template <class T> static void mark_and_push(T* p);
190
191 static inline void push_objarray(oop obj, size_t index);
192
193 static void follow_stack(); // Empty marking stack.
194
195 static void follow_object(oop obj);
196
197 static void follow_array(objArrayOop array, int index);
198
199 static void follow_klass(Klass* klass);
200
201 static void follow_class_loader(ClassLoaderData* cld);
202
203 static int adjust_pointers(oop obj);
204
205 static void preserve_mark(oop p, markOop mark);
206 // Save the mark word so it can be restored later
207 static void adjust_marks(); // Adjust the pointers in the preserved marks table
208 static void restore_marks(); // Restore the marks that we saved in preserve_mark
209
210 static void adjust_marks_helper(Stack<markOop, mtGC>* preserved_mark_stack,
211 Stack<oop, mtGC>* preserved_oop_stack,
212 size_t preserved_count,
213 PreservedMark* preserved_marks);
214 static void restore_marks_helper(Stack<markOop, mtGC>* preserved_mark_stack,
215 Stack<oop, mtGC>* preserved_oop_stack,
216 size_t preserved_count,
217 PreservedMark* preserved_marks,
218 bool should_release);
219
220 template <class T> static inline void adjust_pointer(T* p);
221 };
222
223 class PreservedMark VALUE_OBJ_CLASS_SPEC {
224 private:
225 oop _obj;
226 markOop _mark;
227
228 public:
229 void init(oop obj, markOop mark) {
230 _obj = obj;
231 _mark = mark;
232 }
233
234 void adjust_pointer();
235 void restore();
236 };
237
238 //
239 // CMSParallelFullGC support code
240 //
241 typedef OverflowTaskQueue<oop, mtGC> ObjTaskQueue;
242 typedef GenericTaskQueueSet<ObjTaskQueue, mtGC> ObjTaskQueueSet;
243 typedef OverflowTaskQueue<ObjArrayTask, mtGC> ObjArrayTaskQueue;
244 typedef GenericTaskQueueSet<ObjArrayTaskQueue, mtGC> ObjArrayTaskQueueSet;
245
246 // A region for PMS. A unit of parallelism. It's a fixed-size (1 MB by
247 // default) region (except for the last part of a space) of the space
248 // of the heap.
249 class PMSRegion {
250 public:
251 class RegionDest : public CHeapObj<mtGC> {
252 // CHeapObj because it's accessed by different threads (workers)
253 // and does not have a scoped lifetime. Deallocated by
254 // PMSRegion::cleanup().
255 public:
256 RegionDest() :
257 _from_mr(MemRegion()), _to_space(NULL),
258 _to_space_off(0), _to_space_size(0) {}
259 RegionDest(MemRegion from_mr, CompactibleSpace* to_space,
260 size_t to_space_off, size_t to_space_size) :
261 _from_mr(from_mr), _to_space(to_space),
262 _to_space_off(to_space_off), _to_space_size(to_space_size) {}
263 MemRegion _from_mr; // the address range of the subregion
264 CompactibleSpace* _to_space; // the to-space
265 size_t _to_space_off; // the word offset in the to-space
266 size_t _to_space_size; // the word size in the to_space
267 void log(outputStream* os) {
268 os->print_cr("RegionDest(from:" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT "),"
269 "to:" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT "))",
270 p2i(_from_mr.start()), p2i(_from_mr.end()), _from_mr.byte_size(),
271 p2i(_to_space->bottom() + _to_space_off),
272 p2i(_to_space->bottom() + _to_space_off + _to_space_size),
273 _to_space_size * HeapWordSize);
274 }
275 };
276 enum EvacState { NOT_EVAC, BEING_EVAC, HAS_BEEN_EVAC };
277 // Evacuation state for parallel compaction
278 jbyte _evac_state;
279 // Used for synchronization in the parallel compact phase
280 Monitor* _monitor;
281
282 private:
283 intptr_t _index;
284 MemRegion _mr;
285 CompactibleSpace* _space;
286
287 // The total word size of all the live objects in this region. This
288 // includes all the live objects whose starting addresses are in the
289 // region (but objects' tails can be outside of this region).
290 intptr_t _live_size; // for YG where block_size == obj->size()
291 intptr_t _cfls_live_size; // for CFLS (OG and PG) where
292 // block_size = adjustObjectSize(obj->size())
293
294 HeapWord* _beginning_of_live; // The beginning of the first live object in a region
295 HeapWord* _end_of_live; // The end (exclusive) end of the last live object in a region.
296 HeapWord* _first_moved; // The first live object that moves (for compaction) in a region.
297
298 // Where the live objects in this region are evacuated.
299 GrowableArray<RegionDest*>* _destinations;
300
301 // The (other) regions that must be evacuated before this region can
302 // be evacuated. In parallel compaction, a region cannot be
303 // evacuated until the destination region is evacuated so that no
304 // objects are overwritten before they are copied.
305 GrowableArray<PMSRegion*>* _dependencies;
306
307 public:
308 PMSRegion() : _index(-1), _mr(MemRegion(NULL, (size_t)0)), _evac_state(HAS_BEEN_EVAC),
309 _monitor(NULL), _destinations(NULL), _dependencies(NULL) {}
310 PMSRegion(intptr_t index, HeapWord* start, size_t size, CompactibleSpace* space) :
311 _index(index), _mr(MemRegion(start, size)), _space(space),
312 _live_size(0), _cfls_live_size(0), _destinations(new GrowableArray<RegionDest*>()),
313 _dependencies(new GrowableArray<PMSRegion*>()),
314 _beginning_of_live(start + size), _end_of_live(start), _first_moved(start + size),
315 _evac_state(HAS_BEEN_EVAC) {
316 _monitor = new Monitor(Mutex::barrier, // rank
317 "MarkSweep Region monitor", // name
318 Mutex::_allow_vm_block_flag); // allow_vm_block
319 }
320 // Because this object is resource-allocated, a destructor won't be
321 // called. Use this function to reclaim resources.
322 void cleanup() {
323 if (_monitor != NULL) {
324 delete _monitor;
325 }
326 if (_destinations != NULL) {
327 size_t len = _destinations->length();
328 for (size_t i = 0; i < len; i++) {
329 delete _destinations->at(i);
330 }
331 }
332 }
333 intptr_t index() { return _index; }
334 HeapWord* start() { return _mr.start(); }
335 HeapWord* end() { return _mr.end(); }
336 size_t size() { return _mr.word_size(); }
337 CompactibleSpace* space() { return _space; }
338 size_t live_size() { return (size_t)_live_size; }
339 size_t cfls_live_size() { return (size_t)_cfls_live_size; }
340 inline void add_to_live_size(size_t word_size) {
341 Atomic::add_ptr((intptr_t)word_size, &_live_size);
342 }
343 inline void add_to_cfls_live_size(size_t word_size) {
344 Atomic::add_ptr((intptr_t)word_size, &_cfls_live_size);
345 }
346 GrowableArray<RegionDest*>* destinations() { return _destinations; }
347 void add_destination(MemRegion from_mr, CompactibleSpace* to_space,
348 size_t to_space_off, size_t to_space_size) {
349 RegionDest* d = new RegionDest(from_mr, to_space,
350 to_space_off, to_space_size);
351 _destinations->append(d);
352 }
353 RegionDest* find_destination(HeapWord* addr) {
354 assert(_mr.start() <= addr && addr < _mr.end(), "Must be in the region");
355 size_t len = _destinations->length();
356 for (size_t i = 0; i < len; i++) {
357 RegionDest* d = _destinations->at(i);
358 if (d->_from_mr.contains(addr)) {
359 return d;
360 }
361 }
362 ShouldNotReachHere();
363 return NULL;
364 }
365
366 void log_destinations(outputStream* os) {
367 gclog_or_tty->print("Forwarding region " PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT "): ",
368 p2i(_mr.start()), p2i(_mr.end()), _mr.byte_size());
369 for (int i = 0; i < _destinations->length(); i++) {
370 RegionDest* d = _destinations->at(i);
371 MemRegion dest_mr = MemRegion(d->_to_space->bottom() + d->_to_space_off,
372 d->_to_space_size);
373 gclog_or_tty->print("%d: " PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ") -> "
374 PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ") ",
375 i, p2i(d->_from_mr.start()), p2i(d->_from_mr.end()), d->_from_mr.byte_size(),
376 p2i(dest_mr.start()), p2i(dest_mr.end()), dest_mr.byte_size());
377 }
378 gclog_or_tty->cr();
379 }
380
381 GrowableArray<PMSRegion*>* dependencies() { return _dependencies; }
382
383 void record_stats(HeapWord* beginning_of_live,
384 HeapWord* end_of_live,
385 HeapWord* first_moved) {
386 _beginning_of_live = beginning_of_live;
387 _end_of_live = end_of_live;
388 _first_moved = first_moved;
389 }
390 HeapWord* beginning_of_live() { return _beginning_of_live; }
391 HeapWord* end_of_live() { return _end_of_live; }
392 HeapWord* first_moved() { return _first_moved; }
393 MemRegion live_range() {
394 if (_beginning_of_live < _end_of_live) {
395 return MemRegion(_beginning_of_live, _end_of_live);
396 }
397 return MemRegion();
398 }
399 };
400
401 // An array of PMSRegions. There is one per space. The relevant spaces
402 // under CMS are eden, two survivor spaces (they are the three
403 // ContiguousSpace's in the young gen), the old gen (the entire
404 // generation is one CompactibleFreeListSpace). Note that the
405 // last PMSRegion of a space can be smaller than the standard fixed
406 // size (1 MB by default) (as in a remainder.)
407 class PMSRegionArray {
408 private:
409 CompactibleSpace* _space;
410 MemRegion _mr;
411 PMSRegion* _regions;
412 size_t _region_word_size;
413 size_t _n_regions;
414
415 void reset_regions(MemRegion mr, size_t region_byte_size) {
416 _mr = mr;
417 _region_word_size = region_byte_size/HeapWordSize;
418 assert(region_byte_size % HeapWordSize == 0, "Not word aligned?");
419 size_t mr_size = mr.word_size();
420 _n_regions = mr_size / _region_word_size;
421 if (mr_size % _region_word_size != 0) {
422 // The last region is of an odd size
423 _n_regions++;
424 }
425 _regions = NEW_RESOURCE_ARRAY(PMSRegion, _n_regions);
426 size_t index = 0;
427 HeapWord* p = mr.start();
428 for (; p < mr.end(); p += _region_word_size) {
429 HeapWord* r_start = p;
430 HeapWord* r_end = p + _region_word_size;
431 if (mr.end() < r_end) {
432 // The last region is of an odd size
433 r_end = mr.end();
434 assert(pointer_delta(r_end, r_start) < _region_word_size,
435 "the remainder should be smaller than the standard region size");
436 }
437 _regions[index] = PMSRegion(index, r_start, r_end - r_start, _space);
438 index++;
439 }
440 assert(index == _n_regions, "sanity check");
441 }
442
443 public:
444 PMSRegionArray() : _space(NULL) {}
445 PMSRegionArray(CompactibleSpace* space, size_t region_byte_size) : _space(space) {
446 reset_regions(MemRegion(space->bottom(), space->end()), region_byte_size);
447 }
448 // Because this object is resource-allocated, a destructor won't be
449 // called. Use this function to reclaim resources.
450 void cleanup() {
451 for (size_t i = 0; i < _n_regions; i++) {
452 _regions[i].cleanup();
453 }
454 }
455 inline bool contains(HeapWord* addr) {
456 return _mr.start() <= addr && addr < _mr.end();
457 }
458 CompactibleSpace* space() { return _space; }
459 // The first region
460 PMSRegion* begin() { return &_regions[0]; }
461 // The last region
462 PMSRegion* end() { return &_regions[_n_regions - 1]; }
463 size_t region_word_size() { return _region_word_size; }
464 size_t region_byte_size() { return _region_word_size * HeapWordSize; }
465 size_t n_regions() { return _n_regions; }
466 inline PMSRegion* region_for_index(size_t i) {
467 // Use '<' below so we don't allow the sentinel end region here
468 assert(i < _n_regions, "Out of bounds");
469 return &_regions[i];
470 }
471 inline PMSRegion* region_for_addr(HeapWord* addr) {
472 return region_for_index(addr_to_index(addr));
473 }
474 // Returns the index of the region that contains addr
475 inline size_t addr_to_index(HeapWord* addr) {
476 assert(_mr.start() <= addr && addr < _mr.end(), "Out of bounds");
477 size_t index = pointer_delta(addr, _mr.start()) / _region_word_size;
478 assert(index < _n_regions, "Index out of bounds");
479 return index;
480 }
481 void record_space_live_range_for_compaction() {
482 HeapWord* beginning_of_live = _space->end();
483 HeapWord* end_of_live = _space->bottom();
484 HeapWord* first_moved = _space->end();
485 PMSRegion* first_r = begin();
486 PMSRegion* last_r = end();
487 for (PMSRegion* r = first_r; r <= last_r; r++) {
488 if (r->beginning_of_live() < r->end()) {
489 beginning_of_live = r->beginning_of_live();
490 break;
491 }
492 }
493 for (PMSRegion* r = last_r; r >= first_r; r--) {
494 if (r->start() < r->end_of_live()) {
495 end_of_live = r->end_of_live();
496 break;
497 }
498 }
499 for (PMSRegion* r = first_r; r <= last_r; r++) {
500 if (r->first_moved() < r->end()) {
501 first_moved = r->first_moved();
502 break;
503 }
504 }
505 if (LogCMSParallelFullGC) {
506 gclog_or_tty->print_cr("beginning_of_live=" PTR_FORMAT,
507 p2i(beginning_of_live));
508 gclog_or_tty->print_cr("end_of_live=" PTR_FORMAT,
509 p2i(end_of_live));
510 gclog_or_tty->print_cr("first_moved=" PTR_FORMAT,
511 p2i(first_moved));
512 }
513 assert(beginning_of_live <= first_moved, "sanity check");
514 _space->set_live_range_for_compaction(beginning_of_live,
515 end_of_live,
516 first_moved);
517 }
518 void compute_compact_dependencies();
519 void mark_dense_prefix_as_evacuated() {
520 if (_space->end_of_live() <= _space->first_moved()) {
521 // No objects moved. Nothing to compact.
522 return;
523 }
524 // Mark the 'dense prefix' regions, which have no objects that
525 // will move in this compaction, evacuated. Note that the last
526 // such region may be a destination of an object from a
527 // region. If we don't mark them evacuated here, the compaction
528 // routine (phase 4) may hang because no one will mark the last
529 // such region evacuated.
530 PMSRegion* first_r = region_for_addr(_space->first_moved());
531 for (PMSRegion* r = begin(); r < first_r; r++) {
532 r->_evac_state = PMSRegion::HAS_BEEN_EVAC;
533 }
534 }
535 };
536
537 // A set of PMSRegionArray's. There is one for the entire heap.
538 class PMSRegionArraySet : public ResourceObj {
539 private:
540 class GetSpacesClosure : public SpaceClosure {
541 private:
542 GrowableArray<Space*>* _spaces;
543 public:
544 GetSpacesClosure() {
545 _spaces = new GrowableArray<Space*>();
546 }
547 void do_space(Space* s) {
548 _spaces->append(s);
549 }
550 GrowableArray<Space*>* spaces() { return _spaces; }
551 };
552 class SizeLiveObjectClosure : public BitMapClosure {
553 private:
554 PMSMarkBitMap* _mark_bit_map;
555 Space* _space;
556 size_t _live_size;
557 size_t _cfls_live_size;
558 public:
559 SizeLiveObjectClosure(PMSMarkBitMap* mark_bit_map, Space* space)
560 : _mark_bit_map(mark_bit_map), _space(space),
561 _live_size(0), _cfls_live_size(0) {
562 }
563 bool do_bit(size_t offset);
564 size_t live_size() { return _live_size; }
565 size_t cfls_live_size() { return _cfls_live_size; }
566 };
567
568 enum space_index {
569 EDEN_SPACE = 0,
570 S0_SPACE = 1,
571 S1_SPACE = 2,
572 CMS_SPACE = 3,
573 N_SPACES = 4 // The number of spaces
574 };
575
576 // Spaces and their bottom addresses
577 PMSRegionArray _arrays[N_SPACES];
578 CompactibleSpace* _spaces[N_SPACES];
579 HeapWord* _space_bottoms[N_SPACES];
580
581 public:
582 PMSRegionArraySet() {
583 GetSpacesClosure cl;
584 GenCollectedHeap* gch = GenCollectedHeap::heap();
585 #ifdef ASSERT
586 MemRegion reserved = gch->reserved_region();
587 MemRegion yg_reserved = gch->young_gen()->reserved();
588 MemRegion og_reserved = gch->old_gen()->reserved();
589 if (LogCMSParallelFullGC) {
590 gclog_or_tty->print_cr("reserved: " PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
591 p2i(reserved.start()), p2i(reserved.end()), reserved.byte_size());
592 gclog_or_tty->print_cr("yg_reserved: " PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
593 p2i(yg_reserved.start()), p2i(yg_reserved.end()), yg_reserved.byte_size());
594 gclog_or_tty->print_cr("og_reserved: " PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
595 p2i(og_reserved.start()), p2i(og_reserved.end()), og_reserved.byte_size());
596 }
597 assert(reserved.contains(yg_reserved), "Huh?");
598 assert(reserved.contains(og_reserved), "Huh?");
599 assert(reserved.start() == yg_reserved.start(), "Huh?");
600 assert(yg_reserved.end() == og_reserved.start(), "Huh?");
601 assert(og_reserved.end() == reserved.end(), "Huh?");
602 #endif
603 gch->young_gen()->space_iterate(&cl, true);
604 gch->old_gen()->space_iterate(&cl, true);
605 GrowableArray<Space*>* spaces = cl.spaces();
606 // Five spaces (eden, s0, s1, OG, and PG.)
607 assert(spaces->length() == N_SPACES, "wrong number of spaces");
608 for (size_t i = 0; i < N_SPACES; i++) {
609 Space* s = spaces->at(i);
610 CompactibleSpace* cs = s->toCompactibleSpace();
611 assert(cs != NULL, "Must be a CompactibleSpace");
612 _spaces[i] = cs;
613 }
614 // The spaces come in the ascending address order of 0:eden, 1:s0,
615 // 2:s1, 3:og. Note s0 and s1 can be in a reverse order. Fix it if so.
616 if (_spaces[2]->bottom() < _spaces[1]->bottom()) {
617 CompactibleSpace* tmp = _spaces[2];
618 _spaces[2] = _spaces[1];
619 _spaces[1] = tmp;
620 }
621 #ifdef ASSERT
622 for (size_t i = 0; i < N_SPACES - 1; i++) {
623 assert(_spaces[i]->bottom() < _spaces[i + 1]->bottom(),
624 "The spaces must be in the ascending address order");
625 }
626 #endif
627 for (size_t i = 0; i < N_SPACES; i++) {
628 _arrays[i] = PMSRegionArray(_spaces[i], CMSParallelFullGCHeapRegionSize);
629 _space_bottoms[i] = _spaces[i]->bottom();
630 }
631 }
632
633 // Because this object is resource-allocated, a destructor won't be
634 // called. Use this function to reclaim resources.
635 void cleanup() {
636 for (size_t i = 0; i < N_SPACES; i++) {
637 _arrays[i].cleanup();
638 }
639 }
640
641 CompactibleSpace* cms_space() { return _spaces[CMS_SPACE]; }
642
643 inline PMSRegionArray* region_array_for(HeapWord* addr) {
644 for (size_t i = 0; i < N_SPACES; i++) {
645 PMSRegionArray* ra = &_arrays[i];
646 if (ra->contains(addr)) {
647 return ra;
648 }
649 }
650 return NULL;
651 }
652
653 // A faster region array lookup. This is important for minimizing
654 // marking overhead (MarkSweep::par_mark_object()).
655 inline PMSRegionArray* fast_region_array_for(HeapWord* addr) {
656 PMSRegionArray* res;
657 if (_space_bottoms[CMS_SPACE] <= addr) {
658 res = &_arrays[CMS_SPACE];
659 } else {
660 if (_space_bottoms[S0_SPACE] <= addr) {
661 if (_space_bottoms[S1_SPACE] <= addr) {
662 res = &_arrays[S1_SPACE];
663 } else {
664 res = &_arrays[S0_SPACE];
665 }
666 } else {
667 res = &_arrays[EDEN_SPACE];
668 }
669 }
670 assert(res->contains(addr), "The result region array must contain the address");
671 return res;
672 }
673
674 PMSRegionArray* region_array_for(Space* space) {
675 for (size_t i = 0; i < N_SPACES; i++) {
676 PMSRegionArray* ra = &_arrays[i];
677 if (ra->space() == space) {
678 return ra;
679 }
680 }
681 return NULL;
682 }
683
684 void verify_live_size();
685 };
686
687 // This is an AbstractGangTask used in Phase 1 (marking) of the
688 // parallel mark sweep.
689 class PMSMarkTask : public AbstractGangTask {
690 WorkGang* _workers;
691 ObjTaskQueueSet* _task_queues;
692 ObjArrayTaskQueueSet* _objarray_task_queues;
693 ParallelTaskTerminator _terminator;
694 ParallelTaskTerminator _objarray_terminator;
695 StrongRootsScope* _strong_roots_scope;
696 public:
697 PMSMarkTask(StrongRootsScope *srs, WorkGang* workers,
698 ObjTaskQueueSet* task_queues,
699 ObjArrayTaskQueueSet* objarray_task_queues) :
700 AbstractGangTask("genMarkSweep parallel mark"),
701 _strong_roots_scope(srs), _workers(workers),
702 _task_queues(task_queues), _objarray_task_queues(objarray_task_queues),
703 _terminator(ParallelTaskTerminator(srs->n_threads(), task_queues)),
704 _objarray_terminator(ParallelTaskTerminator(srs->n_threads(), objarray_task_queues)) {}
705 void work(uint worker_id);
706 void do_steal_work(uint worker_id);
707 };
708
709 // This is an AbstractGangTask used in Phase 3 (adjust pointers) of
710 // the parallel mark sweep.
711 class PMSParAdjustTask : public AbstractGangTask {
712 private:
713 WorkGang* _workers;
714 int _n_workers;
715 PMSRegionTaskQueueSet* _task_queues;
716 ParallelTaskTerminator _terminator;
717 BitMapClosure* _cl;
718 public:
719 PMSParAdjustTask(int n_workers, WorkGang* workers,
720 PMSRegionTaskQueueSet* task_queues,
721 BitMapClosure* cl) :
722 AbstractGangTask("genMarkSweep parallel adjust"),
723 _n_workers(n_workers), _workers(workers),
724 _task_queues(task_queues), _cl(cl),
725 _terminator(ParallelTaskTerminator(n_workers, task_queues)) {
726 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC");
727 }
728 void work(uint worker_id);
729 void do_steal_work(uint worker_id);
730 };
731
732 // This is an AbstractGangTask used in Phase 3 (adjust pointers) of
733 // the parallel mark sweep where it adjuts the pointers in the GC
734 // roots.
735 class PMSAdjustRootsTask : public AbstractGangTask {
736 WorkGang* _workers;
737 StrongRootsScope* _strong_roots_scope;
738 public:
739 PMSAdjustRootsTask(StrongRootsScope *srs, WorkGang* workers) :
740 AbstractGangTask("genMarkSweep parallel adjust roots"),
741 _strong_roots_scope(srs), _workers(workers) {}
742 void work(uint worker_id);
743 };
744
745 // This is an AbstractGangTask used in Phase 3 (adjust pointers) of
746 // the parallel mark sweep where it adjusts the pointers in the
747 // 'preserved marks' (the mark header word for each object is
748 // saved/restored before/after the parallel mark sweep.)
749 class PMSParAdjustPreservedMarksTask : public AbstractGangTask {
750 WorkGang* _workers;
751 public:
752 PMSParAdjustPreservedMarksTask(WorkGang* workers) :
753 AbstractGangTask("MarkSweep parallel adjust preserved marks"),
754 _workers(workers) {
755 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC");
756 }
757 void work(uint worker_id);
758 };
759
760 // This is an AbstractGangTask used in Phase 4 (compaction/evacuation)
761 // of the parallel mark sweep.
762 class PMSParEvacTask : public AbstractGangTask {
763 private:
764 WorkGang* _workers;
765 int _n_workers;
766 CompactibleSpace* _space;
767 BitMapClosure* _cl;
768 public:
769 PMSParEvacTask(int n_workers, WorkGang* workers,
770 CompactibleSpace* space, BitMapClosure* cl) :
771 AbstractGangTask("genMarkSweep parallel evac"),
772 _n_workers(n_workers), _workers(workers), _space(space), _cl(cl) {
773 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC");
774 }
775 void work(uint worker_id);
776 };
777
778 // The mark bit map used for the parallel mark sweep.
779 class PMSMarkBitMap : public CHeapObj<mtGC> {
780 private:
781 int _shifter; // We allocate one bit per 2^_shifter in the bit map
782 HeapWord* _start; // The beginning address of the region covered by this mark bit map
783 size_t _size; // The word size of the covered region == The number of bits
784 BitMap _bits; // The underlying BitMap
785 public:
786 PMSMarkBitMap();
787 PMSMarkBitMap(MemRegion underlying_memory);
788
789 inline size_t addr_to_bit(HeapWord* addr) const {
790 assert(addr >= _start, "addr too small");
791 // Note: the addr == _start + _size case is valid for the
792 // (exclusive) right index for the iterate() call.
793 assert(addr <= _start + _size, "addr too large");
794 assert((intptr_t)addr % MinObjAlignmentInBytes == 0,
795 "Non-object-aligned address is suspicious");
796 return pointer_delta(addr, _start) >> _shifter;
797 }
798
799 inline HeapWord* bit_to_addr(size_t bit) const {
800 assert(bit < _bits.size(), "bit out of range");
801 return _start + (bit << _shifter);
802 }
803
804 inline bool mark(oop obj) {
805 size_t bit = addr_to_bit((HeapWord*)obj);
806 return _bits.par_set_bit(bit);
807 }
808
809 inline bool is_marked(oop obj) const {
810 size_t bit = addr_to_bit((HeapWord*)obj);
811 return _bits.at(bit);
812 }
813
814 inline bool iterate(BitMapClosure* cl) { return _bits.iterate(cl); }
815 inline bool iterate(BitMapClosure* cl, size_t left, size_t right) { return _bits.iterate(cl, left, right); }
816 inline size_t count_one_bits() const { return _bits.count_one_bits(); }
817 inline void clear() { return _bits.clear(); }
818 inline bool isAllClear() const { return _bits.is_empty(); }
819 };
820
821 // The BitMapClosure used in Phase 3 (adjust pointers) of the parallel
822 // mark sweep.
823 class PMSAdjustClosure : public BitMapClosure {
824 private:
825 PMSMarkBitMap* _mark_bit_map;
826 public:
827 PMSAdjustClosure(PMSMarkBitMap* mark_bit_map) :
828 _mark_bit_map(mark_bit_map) {
829 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC");
830 }
831 bool do_bit(size_t offset) {
832 HeapWord* addr = _mark_bit_map->bit_to_addr(offset);
833 oop obj = oop(addr);
834 assert(_mark_bit_map->is_marked(obj), "sanity check");
835 const intx interval = PrefetchScanIntervalInBytes;
836 Prefetch::write(addr, interval);
837 MarkSweep::adjust_pointers(obj);
838 return true;
839 }
840 };
841
842 // This BitMapClosure is used in Phase 2 (forwarding) of the parallel
843 // mark sweep for finding a split point of a PMSRegion. A region may
844 // need to be split into multiple subparts in terms of what space they
845 // are evacuated into (copied to) when one destination space is
846 // completely filled.
847 class PMSFindRegionSplitPointClosure : public BitMapClosure {
848 private:
849 PMSMarkBitMap* _mark_bit_map;
850 HeapWord* _compact_top;
851 CompactibleSpace* _dst_space;
852 HeapWord* _dst_space_end;
853 size_t _dst_space_minimum_free_block_size;
854 bool _is_dst_cfls;
855 HeapWord* _split_point;
856 size_t _live_size_up_to_split_point;
857 size_t _cfls_live_size_up_to_split_point;
858 public:
859 PMSFindRegionSplitPointClosure(PMSMarkBitMap* mark_bit_map,
860 HeapWord* compact_top,
861 CompactibleSpace* dst_space)
862 : _mark_bit_map(mark_bit_map), _dst_space(dst_space),
863 _dst_space_end(dst_space->end()),
864 _dst_space_minimum_free_block_size(
865 dst_space->minimum_free_block_size()),
866 _is_dst_cfls(dst_space->isCompactibleFreeListSpace()),
867 _compact_top(compact_top), _split_point(NULL),
868 _live_size_up_to_split_point(0),
869 _cfls_live_size_up_to_split_point(0) {
870 }
871 bool do_bit(size_t offset) {
872 HeapWord* addr = _mark_bit_map->bit_to_addr(offset);
873 oop obj = oop(addr);
874 size_t size = obj->size();
875 size_t cfls_size = CompactibleFreeListSpace::adjustObjectSize(obj->size());
876 size_t dst_size = _is_dst_cfls ? cfls_size : size;
877 HeapWord* dst_space_end = _dst_space_end;
878 if (_compact_top + dst_size > dst_space_end) {
879 _split_point = addr;
880 return false; /* exit out of iteration */
881 }
882 if (_is_dst_cfls &&
883 _compact_top + dst_size +
884 _dst_space_minimum_free_block_size > dst_space_end &&
885 _compact_top + dst_size != _dst_space_end) {
886 /* CFLS cannot leave a residual fragment smaller than */
887 /* MinChunkSize. So split right there. */
888 _split_point = addr;
889 return false;
890 }
891 _compact_top += dst_size;
892 _live_size_up_to_split_point += size;
893 _cfls_live_size_up_to_split_point += cfls_size;
894 return true; /* continue iteration */
895 }
896 HeapWord* split_point() { return _split_point; }
897 HeapWord* compact_top() { return _compact_top; }
898 size_t live_size_up_to_split_point() {
899 return _live_size_up_to_split_point;
900 }
901 size_t cfls_live_size_up_to_split_point() {
902 return _cfls_live_size_up_to_split_point;
903 }
904 };
905
906 #define pms_cfls_obj_size(q) CompactibleFreeListSpace::adjustObjectSize(oop(q)->size())
907
908 // This macro defines specialized support code for parallel mark
909 // sweep. It's defined as a macro to de-virtualize (specialize) the
910 // object size routine (obj_size) for ContiguousSpace (space.cpp) and
911 // CompactibleFreeListSpace (compactibleFreeListSpace.cpp),
912 // respectively, like SCAN_AND_xxx in space.hpp.
913 //
914 // This macro contains three classes and two functions related to 'class_name'.
915 //
916 // The three classes are:
917 // PMSPerRegionForwardClosure_##class_name (used in the forward phase)
918 // PMSForwardTask_##class_name (used in the forward phase)
919 // PMSCompactClosure_##class_name (used in the compact phase)
920 //
921 // The two functions are:
922 // class_name::pms_prepare_for_compaction_work() (used in the forward phase)
923 // class_name::pms_compact_work() (used in the compact phase)
924 //
925 // where 'class_name' is either ContiguousSpace or CompactibleFreeListSpace.
926 //
927 #define DECLARE_PMS_SPECIALIZED_CODE(class_name, obj_size) \
928 /* This is the BitMapClosure used for Phase 2 (forwarding) of PMS. */ \
929 /* This is used to forward (objects in) a region (PMSRegion) at a time */ \
930 /* for parallelism. */ \
931 class PMSPerRegionForwardClosure_##class_name : public BitMapClosure { \
932 private: \
933 PMSMarkBitMap* _mark_bit_map; \
934 PMSRegion* _region; \
935 PMSRegion::RegionDest* _dest; \
936 CompactibleSpace* _dest_space; \
937 bool _is_dest_cfls; \
938 HeapWord* _compact_top; \
939 HeapWord* _beginning_of_live; /* the first live object */ \
940 HeapWord* _end_of_live; /* right after the last live object */ \
941 HeapWord* _first_moved; /* the first moved (live) object */ \
942 /* where the compaction phase */ \
943 /* starts from */ \
944 bool _beginning_of_live_set; \
945 bool _first_moved_set; \
946 void update_dest(HeapWord* addr) { \
947 _dest = _region->find_destination(addr); \
948 assert(_dest->_from_mr.contains(addr), "sanity check"); \
949 _dest_space = _dest->_to_space; \
950 _is_dest_cfls = _dest_space->isCompactibleFreeListSpace(); \
951 _compact_top = _dest_space->bottom() + _dest->_to_space_off; \
952 } \
953 public: \
954 PMSPerRegionForwardClosure_##class_name(PMSMarkBitMap* mark_bit_map, \
955 PMSRegion* region) : \
956 _mark_bit_map(mark_bit_map), _region(region), \
957 _beginning_of_live(region->end()), _end_of_live(region->start()), \
958 _first_moved(region->end()), _beginning_of_live_set(false), \
959 _first_moved_set(false) { \
960 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC"); \
961 update_dest(region->start()); \
962 } \
963 HeapWord* compact_top() { return _compact_top; } \
964 HeapWord* beginning_of_live() { return _beginning_of_live; } \
965 HeapWord* end_of_live() { return _end_of_live; } \
966 HeapWord* first_moved() { return _first_moved; } \
967 bool do_bit(size_t offset) { \
968 HeapWord* addr = _mark_bit_map->bit_to_addr(offset); \
969 assert(_region->start() <= addr && addr < _region->end(), "out of region"); \
970 if (_dest->_from_mr.end() <= addr) { \
971 update_dest(addr); \
972 } \
973 assert(_dest->_from_mr.contains(addr), "out of bounds"); \
974 if (!_beginning_of_live_set) { \
975 _beginning_of_live_set = true; \
976 _beginning_of_live = addr; \
977 /* Tighten the from_mr to the live portion only */ \
978 _dest->_from_mr = MemRegion(addr, _dest->_from_mr.end()); \
979 } \
980 oop obj = oop(addr); \
981 assert(_mark_bit_map->is_marked(obj), "sanity check"); \
982 assert((intptr_t)addr % MinObjAlignmentInBytes == 0, "obj alignemnt check");\
983 const intx interval = PrefetchScanIntervalInBytes; \
984 Prefetch::write(addr, interval); \
985 size_t ssize = obj_size(addr); \
986 size_t dsize = _is_dest_cfls ? pms_cfls_obj_size(obj) : obj->size(); \
987 if (addr != _compact_top) { \
988 obj->forward_to(oop(_compact_top)); \
989 assert(MarkSweep::is_object_marked(obj), \
990 "encoding the pointer should preserve the mark"); \
991 } else { \
992 obj->init_mark(); \
993 assert(obj->forwardee() == NULL, "should be forwarded to NULL"); \
994 } \
995 /* Update the BOT. Make sure this is MT-safe */ \
996 _dest_space->cross_threshold(_compact_top, _compact_top + dsize); \
997 _compact_top += dsize; \
998 _end_of_live = addr + ssize; \
999 /* Tighten the from_mr only to the live portion. Especially, */ \
1000 /* for the last object whose end sticks out of the end of the region */ \
1001 /* we need to update the end of from_mr. */ \
1002 if (_region->end() < _end_of_live) { \
1003 _dest->_from_mr = MemRegion(_dest->_from_mr.start(), _end_of_live); \
1004 } \
1005 assert(_end_of_live <= _region->space()->end(), "Out of space bound"); \
1006 if (!_first_moved_set && obj->forwardee() != NULL) { \
1007 /* Non-moving object's forwarding pointer is set to NULL */ \
1008 _first_moved_set = true; \
1009 _first_moved = addr; \
1010 } \
1011 return true; \
1012 } \
1013 void record() { \
1014 _region->record_stats(_beginning_of_live, _end_of_live, _first_moved); \
1015 } \
1016 }; \
1017 /* This is an AbstractGangTask used for Phase 2 (forwarding) of PMS. */ \
1018 /* This is used to forward (objects in) a region (PMSRegion) at a time */ \
1019 /* for parallelism. */ \
1020 class PMSForwardTask_##class_name : public AbstractGangTask { \
1021 private: \
1022 typedef OverflowTaskQueue<PMSRegion*, mtGC> PMSRegionTaskQueue; \
1023 typedef GenericTaskQueueSet<PMSRegionTaskQueue, mtGC> PMSRegionTaskQueueSet; \
1024 WorkGang* _workers; \
1025 int _n_workers; \
1026 PMSRegionTaskQueueSet* _task_queues; \
1027 ParallelTaskTerminator _terminator; \
1028 PMSMarkBitMap* _mark_bit_map; \
1029 public: \
1030 PMSForwardTask_##class_name(int n_workers, WorkGang* workers, \
1031 PMSRegionTaskQueueSet* task_queues) : \
1032 AbstractGangTask("genMarkSweep parallel forward"), \
1033 _n_workers(n_workers), _workers(workers), \
1034 _task_queues(task_queues), \
1035 _terminator(ParallelTaskTerminator(n_workers, task_queues)), \
1036 _mark_bit_map(MarkSweep::pms_mark_bit_map()) { \
1037 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC"); \
1038 } \
1039 void work(uint worker_id) { \
1040 elapsedTimer timer; \
1041 ResourceMark rm; \
1042 HandleMark hm; \
1043 if (LogCMSParallelFullGC) { \
1044 timer.start(); \
1045 gclog_or_tty->print_cr("Parallel forward worker %d started...", \
1046 worker_id); \
1047 } \
1048 PMSRegionTaskQueue* task_queue = _task_queues->queue(worker_id); \
1049 GenCollectedHeap* gch = GenCollectedHeap::heap(); \
1050 elapsedTimer timer1; \
1051 if (LogCMSParallelFullGC) { \
1052 timer1.start(); \
1053 } \
1054 PMSRegion* r; \
1055 do { \
1056 while (task_queue->pop_overflow(r)) { \
1057 PMSPerRegionForwardClosure_##class_name cl(_mark_bit_map, r); \
1058 _mark_bit_map->iterate(&cl, \
1059 _mark_bit_map->addr_to_bit(r->start()), \
1060 _mark_bit_map->addr_to_bit(r->end())); \
1061 cl.record(); \
1062 } \
1063 while (task_queue->pop_local(r)) { \
1064 PMSPerRegionForwardClosure_##class_name cl(_mark_bit_map, r); \
1065 _mark_bit_map->iterate(&cl, \
1066 _mark_bit_map->addr_to_bit(r->start()), \
1067 _mark_bit_map->addr_to_bit(r->end())); \
1068 cl.record(); \
1069 } \
1070 } while (!task_queue->is_empty()); \
1071 if (LogCMSParallelFullGC) { \
1072 timer1.stop(); \
1073 gclog_or_tty->print_cr("Parallel forward worker %d finished scanning " \
1074 "(%3.5f sec)", \
1075 worker_id, timer1.seconds()); \
1076 } \
1077 do_steal_work(worker_id); \
1078 if (LogCMSParallelFullGC) { \
1079 timer.stop(); \
1080 gclog_or_tty->print_cr("Parallel forward worker %d finished (%3.5f sec)", \
1081 worker_id, timer.seconds()); \
1082 } \
1083 } \
1084 \
1085 void do_steal_work(uint worker_id) { \
1086 elapsedTimer timer; \
1087 if (LogCMSParallelFullGC) { \
1088 timer.start(); \
1089 } \
1090 int seed = 17; \
1091 int num_steals = 0; \
1092 PMSRegionTaskQueue* task_queue = _task_queues->queue(worker_id); \
1093 assert(task_queue == _task_queues->queue(worker_id), "Sanity check"); \
1094 PMSRegion* r; \
1095 do { \
1096 assert(task_queue->is_empty(), \
1097 "Task queue should be empty before work stealing"); \
1098 while (_task_queues->steal(worker_id, &seed, r)) { \
1099 num_steals++; \
1100 PMSPerRegionForwardClosure_##class_name cl(_mark_bit_map, r); \
1101 _mark_bit_map->iterate(&cl, \
1102 _mark_bit_map->addr_to_bit(r->start()), \
1103 _mark_bit_map->addr_to_bit(r->end())); \
1104 cl.record(); \
1105 } \
1106 } while (!_terminator.offer_termination()); \
1107 assert(task_queue->is_empty(), "Check my task queue is empty again"); \
1108 if (LogCMSParallelFullGC) { \
1109 timer.stop(); \
1110 gclog_or_tty->print_cr( \
1111 "Parallel forward worker %d finished work-stealing " \
1112 "(%d steals, %3.5f sec)", \
1113 worker_id, num_steals, timer.seconds()); \
1114 } \
1115 } \
1116 }; \
1117 \
1118 /* This is the BitMapClosure used for Phase 4 (compaction/evacuation) of PMS.*/ \
1119 /* This is used to copy objects. */ \
1120 class PMSCompactClosure_##class_name : public BitMapClosure { \
1121 private: \
1122 PMSMarkBitMap* _mark_bit_map; \
1123 public: \
1124 PMSCompactClosure_##class_name(PMSMarkBitMap* mark_bit_map) : \
1125 _mark_bit_map(mark_bit_map) { \
1126 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC"); \
1127 } \
1128 bool do_bit(size_t offset) { \
1129 HeapWord* addr = _mark_bit_map->bit_to_addr(offset); \
1130 oop obj = oop(addr); \
1131 assert(_mark_bit_map->is_marked(obj), "sanity check"); \
1132 const intx scan_interval = PrefetchScanIntervalInBytes; \
1133 const intx copy_interval = PrefetchCopyIntervalInBytes; \
1134 Prefetch::read(addr, scan_interval); \
1135 size_t size = obj_size(addr); \
1136 HeapWord* compaction_top = (HeapWord*)obj->forwardee(); \
1137 assert(compaction_top != NULL, "Non-moving object should not be seen here");\
1138 Prefetch::write(compaction_top, copy_interval); \
1139 assert(addr != compaction_top, "everything in this pass should be moving"); \
1140 Copy::aligned_conjoint_words(addr, compaction_top, size); \
1141 oop(compaction_top)->init_mark(); \
1142 assert(oop(compaction_top)->klass() != NULL, "should have a class"); \
1143 return true; \
1144 } \
1145 }; \
1146 \
1147 /* Parallel version of prepare_for_compaction() used for Phase 2 (forwarding)*/ \
1148 /* of PMS. */ \
1149 void class_name::pms_prepare_for_compaction_work(CompactPoint* cp) { \
1150 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC"); \
1151 HeapWord* compact_top; \
1152 set_compaction_top(bottom()); \
1153 \
1154 if (cp->space == NULL) { \
1155 assert(cp->gen != NULL, "need a generation"); \
1156 assert(cp->threshold == NULL, "just checking"); \
1157 assert(cp->gen->first_compaction_space() == this, "just checking"); \
1158 cp->space = cp->gen->first_compaction_space(); \
1159 compact_top = cp->space->bottom(); \
1160 cp->space->set_compaction_top(compact_top); \
1161 cp->threshold = cp->space->initialize_threshold(); \
1162 } else { \
1163 compact_top = cp->space->compaction_top(); \
1164 } \
1165 \
1166 PMSMarkBitMap* mark_bit_map = MarkSweep::pms_mark_bit_map(); \
1167 GenCollectedHeap* gch = GenCollectedHeap::heap(); \
1168 PMSRegionTaskQueueSet* task_queues = MarkSweep::pms_region_task_queues(); \
1169 WorkGang* workers = gch->workers(); \
1170 int n_workers = workers->total_workers(); \
1171 PMSRegionArraySet* region_array_set = MarkSweep::pms_region_array_set(); \
1172 PMSRegionArray* regions = region_array_set->region_array_for(this); \
1173 assert(regions != NULL, "Must be non-null"); \
1174 \
1175 PMSRegion* first_r = regions->begin(); \
1176 PMSRegion* last_r = regions->end(); /* Inclusive */ \
1177 int i = 0; \
1178 \
1179 for (PMSRegion* r = first_r; r <= last_r; r++) { \
1180 size_t live_size = r->live_size(); \
1181 size_t cfls_live_size = r->cfls_live_size(); \
1182 if (live_size == 0) { \
1183 r->_evac_state = PMSRegion::HAS_BEEN_EVAC; \
1184 assert(cfls_live_size == 0, "cfls_live_size must be zero, too"); \
1185 continue; \
1186 } \
1187 r->_evac_state = PMSRegion::NOT_EVAC; \
1188 bool is_cfls = cp->space->isCompactibleFreeListSpace(); \
1189 size_t size = is_cfls ? cfls_live_size : live_size; \
1190 size_t compaction_max_size = pointer_delta(cp->space->end(), compact_top); \
1191 HeapWord* split_point = r->start(); \
1192 /* If the current compaction space is not large enough or the */ \
1193 /* remaining fragment is smaller than MinChunkSize (which is not */ \
1194 /* allowed for the CMS space), we need to use the next */ \
1195 /* compaction space and split the region. */ \
1196 while (size > compaction_max_size || \
1197 (size + cp->space->minimum_free_block_size() > compaction_max_size &&\
1198 size != compaction_max_size)) { \
1199 /* First split the region */ \
1200 HeapWord* prev_compact_top = compact_top; \
1201 HeapWord* prev_split_point = split_point; \
1202 PMSFindRegionSplitPointClosure cl(mark_bit_map, compact_top, cp->space); \
1203 mark_bit_map->iterate(&cl, \
1204 mark_bit_map->addr_to_bit(split_point), \
1205 mark_bit_map->addr_to_bit(r->end())); \
1206 split_point = cl.split_point(); \
1207 assert(split_point != NULL, "Region must be split"); \
1208 compact_top = cl.compact_top(); \
1209 size_t live_size_up_to_split_point = cl.live_size_up_to_split_point(); \
1210 size_t cfls_live_size_up_to_split_point = \
1211 cl.cfls_live_size_up_to_split_point(); \
1212 if (prev_split_point != split_point) { \
1213 /* This is for the first split piece of the region */ \
1214 r->add_destination(MemRegion(prev_split_point, split_point), \
1215 cp->space, prev_compact_top - cp->space->bottom(), \
1216 is_cfls ? cfls_live_size_up_to_split_point : \
1217 live_size_up_to_split_point); \
1218 } else { \
1219 assert(compact_top == prev_compact_top, \
1220 "the space didn't accommodate even one object"); \
1221 } \
1222 /* switch to next compaction space */ \
1223 cp->space->set_compaction_top(compact_top); \
1224 cp->space = cp->space->next_compaction_space(); \
1225 if (cp->space == NULL) { \
1226 assert(gch->is_old_gen(cp->gen), "compaction must succeed"); \
1227 cp->gen = gch->young_gen(); \
1228 cp->space = cp->gen->first_compaction_space(); \
1229 assert(cp->space != NULL, \
1230 "generation must have a first compaction space"); \
1231 } \
1232 is_cfls = cp->space->isCompactibleFreeListSpace(); \
1233 compact_top = cp->space->bottom(); \
1234 cp->space->set_compaction_top(compact_top); \
1235 cp->threshold = cp->space->initialize_threshold(); \
1236 compaction_max_size = pointer_delta(cp->space->end(), compact_top); \
1237 live_size -= live_size_up_to_split_point; \
1238 cfls_live_size -= cfls_live_size_up_to_split_point; \
1239 size = is_cfls ? cfls_live_size : live_size; \
1240 } \
1241 assert(size <= compaction_max_size, \
1242 "The whole region (if no split) " \
1243 "or the last split piece must fit the current compact space"); \
1244 /* This is for the whole region (if no split) or the last split piece */ \
1245 /* of the region. Note the from_mr will be 'tightened' later in */ \
1246 /* PMSPerRegionForwardClosure_### so that it will be the */ \
1247 /* smallest memory range that contains only the live portion of the */ \
1248 /* region. Note that the last object's tail can stick out of the region.*/ \
1249 r->add_destination(MemRegion(split_point, r->end()), \
1250 cp->space, compact_top - cp->space->bottom(), size); \
1251 compact_top += size; \
1252 assert(compact_top <= cp->space->end(), "Must fit"); \
1253 assert((intptr_t)compact_top % MinObjAlignmentInBytes == 0, \
1254 "obj alignemnt check"); \
1255 task_queues->queue(i)->push(r); \
1256 i = (i + 1) % n_workers; \
1257 if (LogCMSParallelFullGC) { \
1258 r->log_destinations(gclog_or_tty); \
1259 } \
1260 } \
1261 /* save the compaction_top of the compaction space. */ \
1262 cp->space->set_compaction_top(compact_top); \
1263 PMSForwardTask_##class_name tsk(n_workers, workers, task_queues); \
1264 GCTraceTime tm1("par-forward", \
1265 (PrintGC && Verbose) || LogCMSParallelFullGC, \
1266 true, NULL, GCId::peek()); \
1267 if (n_workers > 1) { \
1268 workers->run_task(&tsk); \
1269 } else { \
1270 tsk.work(0); \
1271 } \
1272 /* Save the global info from the regions. Note _first_dead is unused. */ \
1273 assert(regions->space() == this, "sanity check"); \
1274 /* TODO: get rid of the parameter below */ \
1275 regions->record_space_live_range_for_compaction(); \
1276 regions->compute_compact_dependencies(); \
1277 regions->mark_dense_prefix_as_evacuated(); \
1278 } \
1279 \
1280 /* Parallel version of compact() used for Phase 4 (compaction/evacuation) */ \
1281 /* of PMS. */ \
1282 void class_name::pms_compact_work() { \
1283 assert(CMSParallelFullGC, "Used only if CMSParallelFullGC"); \
1284 if (_first_moved < _end_of_live) { /* Unless there is no moving live object*/ \
1285 PMSMarkBitMap* mark_bit_map = MarkSweep::pms_mark_bit_map(); \
1286 PMSCompactClosure_##class_name compact_cl(mark_bit_map); \
1287 GenCollectedHeap* gch = GenCollectedHeap::heap(); \
1288 WorkGang* workers = gch->workers(); \
1289 int n_workers = workers->total_workers(); \
1290 PMSParEvacTask tsk(n_workers, workers, this, &compact_cl); \
1291 GCTraceTime tm1("par-compact", \
1292 (PrintGC && Verbose) || LogCMSParallelFullGC, \
1293 true, NULL, GCId::peek()); \
1294 if (n_workers > 1) { \
1295 workers->run_task(&tsk); \
1296 } else { \
1297 tsk.work(0); \
1298 } \
1299 } \
1300 /* Let's remember if we were empty before we did the compaction. */ \
1301 bool was_empty = used_region().is_empty(); \
1302 /* Reset space after compaction is complete */ \
1303 reset_after_compaction(); \
1304 /* We do this clear, below, since it has overloaded meanings for some */ \
1305 /* space subtypes. For example, OffsetTableContigSpace's that were */ \
1306 /* compacted into will have had their offset table thresholds updated */ \
1307 /* continuously, but those that weren't need to have their thresholds */ \
1308 /* re-initialized. Also mangles unused area for debugging. */ \
1309 if (used_region().is_empty()) { \
1310 if (!was_empty) clear(SpaceDecorator::Mangle); \
1311 } else { \
1312 if (ZapUnusedHeapArea) mangle_unused_area(); \
1313 } \
1314 }
1315
1316 class PMSRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
1317 private:
1318 ObjTaskQueueSet* _task_queues;
1319 ObjArrayTaskQueueSet* _objarray_task_queues;
1320 public:
1321 PMSRefProcTaskExecutor(ObjTaskQueueSet* task_queues,
1322 ObjArrayTaskQueueSet* objarray_task_queues)
1323 : _task_queues(task_queues), _objarray_task_queues(objarray_task_queues) {}
1324
1325 // Executes a task using worker threads.
1326 virtual void execute(ProcessTask& task);
1327 virtual void execute(EnqueueTask& task);
1328 };
1329
1330 #endif // SHARE_VM_GC_SERIAL_MARKSWEEP_HPP
|