1 /*
2 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 class ParallelScavengeHeap;
26 class PSAdaptiveSizePolicy;
27 class PSYoungGen;
28 class PSOldGen;
29 class PSPermGen;
30 class ParCompactionManager;
31 class ParallelTaskTerminator;
32 class PSParallelCompact;
33 class GCTaskManager;
34 class GCTaskQueue;
35 class PreGCValues;
36 class MoveAndUpdateClosure;
37 class RefProcTaskExecutor;
38
39 // The SplitInfo class holds the information needed to 'split' a source region
40 // so that the live data can be copied to two destination *spaces*. Normally,
41 // all the live data in a region is copied to a single destination space (e.g.,
42 // everything live in a region in eden is copied entirely into the old gen).
43 // However, when the heap is nearly full, all the live data in eden may not fit
44 // into the old gen. Copying only some of the regions from eden to old gen
45 // requires finding a region that does not contain a partial object (i.e., no
46 // live object crosses the region boundary) somewhere near the last object that
47 // does fit into the old gen. Since it's not always possible to find such a
48 // region, splitting is necessary for predictable behavior.
49 //
50 // A region is always split at the end of the partial object. This avoids
51 // additional tests when calculating the new location of a pointer, which is a
52 // very hot code path. The partial object and everything to its left will be
53 // copied to another space (call it dest_space_1). The live data to the right
54 // of the partial object will be copied either within the space itself, or to a
55 // different destination space (distinct from dest_space_1).
56 //
57 // Split points are identified during the summary phase, when region
58 // destinations are computed: data about the split, including the
59 // partial_object_size, is recorded in a SplitInfo record and the
60 // partial_object_size field in the summary data is set to zero. The zeroing is
61 // possible (and necessary) since the partial object will move to a different
62 // destination space than anything to its right, thus the partial object should
63 // not affect the locations of any objects to its right.
64 //
65 // The recorded data is used during the compaction phase, but only rarely: when
66 // the partial object on the split region will be copied across a destination
67 // region boundary. This test is made once each time a region is filled, and is
68 // a simple address comparison, so the overhead is negligible (see
69 // PSParallelCompact::first_src_addr()).
70 //
71 // Notes:
72 //
73 // Only regions with partial objects are split; a region without a partial
74 // object does not need any extra bookkeeping.
75 //
76 // At most one region is split per space, so the amount of data required is
77 // constant.
78 //
79 // A region is split only when the destination space would overflow. Once that
80 // happens, the destination space is abandoned and no other data (even from
81 // other source spaces) is targeted to that destination space. Abandoning the
82 // destination space may leave a somewhat large unused area at the end, if a
83 // large object caused the overflow.
84 //
85 // Future work:
86 //
87 // More bookkeeping would be required to continue to use the destination space.
88 // The most general solution would allow data from regions in two different
89 // source spaces to be "joined" in a single destination region. At the very
90 // least, additional code would be required in next_src_region() to detect the
91 // join and skip to an out-of-order source region. If the join region was also
92 // the last destination region to which a split region was copied (the most
93 // likely case), then additional work would be needed to get fill_region() to
94 // stop iteration and switch to a new source region at the right point. Basic
95 // idea would be to use a fake value for the top of the source space. It is
96 // doable, if a bit tricky.
97 //
98 // A simpler (but less general) solution would fill the remainder of the
99 // destination region with a dummy object and continue filling the next
100 // destination region.
101
102 class SplitInfo
103 {
104 public:
105 // Return true if this split info is valid (i.e., if a split has been
106 // recorded). The very first region cannot have a partial object and thus is
107 // never split, so 0 is the 'invalid' value.
108 bool is_valid() const { return _src_region_idx > 0; }
109
110 // Return true if this split holds data for the specified source region.
111 inline bool is_split(size_t source_region) const;
112
113 // The index of the split region, the size of the partial object on that
114 // region and the destination of the partial object.
115 size_t src_region_idx() const { return _src_region_idx; }
116 size_t partial_obj_size() const { return _partial_obj_size; }
117 HeapWord* destination() const { return _destination; }
118
119 // The destination count of the partial object referenced by this split
120 // (either 1 or 2). This must be added to the destination count of the
121 // remainder of the source region.
122 unsigned int destination_count() const { return _destination_count; }
123
124 // If a word within the partial object will be written to the first word of a
125 // destination region, this is the address of the destination region;
126 // otherwise this is NULL.
127 HeapWord* dest_region_addr() const { return _dest_region_addr; }
128
129 // If a word within the partial object will be written to the first word of a
130 // destination region, this is the address of that word within the partial
131 // object; otherwise this is NULL.
132 HeapWord* first_src_addr() const { return _first_src_addr; }
133
134 // Record the data necessary to split the region src_region_idx.
135 void record(size_t src_region_idx, size_t partial_obj_size,
136 HeapWord* destination);
137
138 void clear();
139
140 DEBUG_ONLY(void verify_clear();)
141
142 private:
143 size_t _src_region_idx;
144 size_t _partial_obj_size;
145 HeapWord* _destination;
146 unsigned int _destination_count;
147 HeapWord* _dest_region_addr;
148 HeapWord* _first_src_addr;
149 };
150
151 inline bool SplitInfo::is_split(size_t region_idx) const
152 {
153 return _src_region_idx == region_idx && is_valid();
154 }
155
156 class SpaceInfo
157 {
158 public:
159 MutableSpace* space() const { return _space; }
160
161 // Where the free space will start after the collection. Valid only after the
162 // summary phase completes.
163 HeapWord* new_top() const { return _new_top; }
164
165 // Allows new_top to be set.
166 HeapWord** new_top_addr() { return &_new_top; }
167
168 // Where the smallest allowable dense prefix ends (used only for perm gen).
169 HeapWord* min_dense_prefix() const { return _min_dense_prefix; }
170
171 // Where the dense prefix ends, or the compacted region begins.
172 HeapWord* dense_prefix() const { return _dense_prefix; }
173
174 // The start array for the (generation containing the) space, or NULL if there
175 // is no start array.
176 ObjectStartArray* start_array() const { return _start_array; }
177
178 SplitInfo& split_info() { return _split_info; }
179
180 void set_space(MutableSpace* s) { _space = s; }
181 void set_new_top(HeapWord* addr) { _new_top = addr; }
182 void set_min_dense_prefix(HeapWord* addr) { _min_dense_prefix = addr; }
183 void set_dense_prefix(HeapWord* addr) { _dense_prefix = addr; }
184 void set_start_array(ObjectStartArray* s) { _start_array = s; }
185
186 void publish_new_top() const { _space->set_top(_new_top); }
187
188 private:
189 MutableSpace* _space;
190 HeapWord* _new_top;
191 HeapWord* _min_dense_prefix;
192 HeapWord* _dense_prefix;
193 ObjectStartArray* _start_array;
194 SplitInfo _split_info;
195 };
196
197 class ParallelCompactData
198 {
199 public:
200 // Sizes are in HeapWords, unless indicated otherwise.
201 static const size_t Log2RegionSize;
202 static const size_t RegionSize;
203 static const size_t RegionSizeBytes;
204
205 // Mask for the bits in a size_t to get an offset within a region.
206 static const size_t RegionSizeOffsetMask;
207 // Mask for the bits in a pointer to get an offset within a region.
208 static const size_t RegionAddrOffsetMask;
209 // Mask for the bits in a pointer to get the address of the start of a region.
210 static const size_t RegionAddrMask;
211
212 class RegionData
213 {
214 public:
215 // Destination address of the region.
216 HeapWord* destination() const { return _destination; }
217
218 // The first region containing data destined for this region.
219 size_t source_region() const { return _source_region; }
220
221 // The object (if any) starting in this region and ending in a different
222 // region that could not be updated during the main (parallel) compaction
223 // phase. This is different from _partial_obj_addr, which is an object that
224 // extends onto a source region. However, the two uses do not overlap in
225 // time, so the same field is used to save space.
226 HeapWord* deferred_obj_addr() const { return _partial_obj_addr; }
227
228 // The starting address of the partial object extending onto the region.
229 HeapWord* partial_obj_addr() const { return _partial_obj_addr; }
230
231 // Size of the partial object extending onto the region (words).
232 size_t partial_obj_size() const { return _partial_obj_size; }
233
234 // Size of live data that lies within this region due to objects that start
235 // in this region (words). This does not include the partial object
236 // extending onto the region (if any), or the part of an object that extends
237 // onto the next region (if any).
238 size_t live_obj_size() const { return _dc_and_los & los_mask; }
239
240 // Total live data that lies within the region (words).
241 size_t data_size() const { return partial_obj_size() + live_obj_size(); }
242
243 // The destination_count is the number of other regions to which data from
244 // this region will be copied. At the end of the summary phase, the valid
245 // values of destination_count are
246 //
247 // 0 - data from the region will be compacted completely into itself, or the
248 // region is empty. The region can be claimed and then filled.
249 // 1 - data from the region will be compacted into 1 other region; some
250 // data from the region may also be compacted into the region itself.
251 // 2 - data from the region will be copied to 2 other regions.
252 //
253 // During compaction as regions are emptied, the destination_count is
254 // decremented (atomically) and when it reaches 0, it can be claimed and
255 // then filled.
256 //
257 // A region is claimed for processing by atomically changing the
258 // destination_count to the claimed value (dc_claimed). After a region has
259 // been filled, the destination_count should be set to the completed value
260 // (dc_completed).
261 inline uint destination_count() const;
262 inline uint destination_count_raw() const;
263
264 // The location of the java heap data that corresponds to this region.
265 inline HeapWord* data_location() const;
266
267 // The highest address referenced by objects in this region.
268 inline HeapWord* highest_ref() const;
269
270 // Whether this region is available to be claimed, has been claimed, or has
271 // been completed.
272 //
273 // Minor subtlety: claimed() returns true if the region is marked
274 // completed(), which is desirable since a region must be claimed before it
275 // can be completed.
276 bool available() const { return _dc_and_los < dc_one; }
277 bool claimed() const { return _dc_and_los >= dc_claimed; }
278 bool completed() const { return _dc_and_los >= dc_completed; }
279
280 // These are not atomic.
281 void set_destination(HeapWord* addr) { _destination = addr; }
282 void set_source_region(size_t region) { _source_region = region; }
283 void set_deferred_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; }
284 void set_partial_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; }
285 void set_partial_obj_size(size_t words) {
286 _partial_obj_size = (region_sz_t) words;
287 }
288
289 inline void set_destination_count(uint count);
290 inline void set_live_obj_size(size_t words);
291 inline void set_data_location(HeapWord* addr);
292 inline void set_completed();
293 inline bool claim_unsafe();
294
295 // These are atomic.
296 inline void add_live_obj(size_t words);
297 inline void set_highest_ref(HeapWord* addr);
298 inline void decrement_destination_count();
299 inline bool claim();
300
301 private:
302 // The type used to represent object sizes within a region.
303 typedef uint region_sz_t;
304
305 // Constants for manipulating the _dc_and_los field, which holds both the
306 // destination count and live obj size. The live obj size lives at the
307 // least significant end so no masking is necessary when adding.
308 static const region_sz_t dc_shift; // Shift amount.
309 static const region_sz_t dc_mask; // Mask for destination count.
310 static const region_sz_t dc_one; // 1, shifted appropriately.
311 static const region_sz_t dc_claimed; // Region has been claimed.
312 static const region_sz_t dc_completed; // Region has been completed.
313 static const region_sz_t los_mask; // Mask for live obj size.
314
315 HeapWord* _destination;
316 size_t _source_region;
317 HeapWord* _partial_obj_addr;
318 region_sz_t _partial_obj_size;
319 region_sz_t volatile _dc_and_los;
320 #ifdef ASSERT
321 // These enable optimizations that are only partially implemented. Use
322 // debug builds to prevent the code fragments from breaking.
323 HeapWord* _data_location;
324 HeapWord* _highest_ref;
325 #endif // #ifdef ASSERT
326
327 #ifdef ASSERT
328 public:
329 uint _pushed; // 0 until region is pushed onto a worker's stack
330 private:
331 #endif
332 };
333
334 public:
335 ParallelCompactData();
336 bool initialize(MemRegion covered_region);
337
338 size_t region_count() const { return _region_count; }
339
340 // Convert region indices to/from RegionData pointers.
341 inline RegionData* region(size_t region_idx) const;
342 inline size_t region(const RegionData* const region_ptr) const;
343
344 // Returns true if the given address is contained within the region
345 bool region_contains(size_t region_index, HeapWord* addr);
346
347 void add_obj(HeapWord* addr, size_t len);
348 void add_obj(oop p, size_t len) { add_obj((HeapWord*)p, len); }
349
350 // Fill in the regions covering [beg, end) so that no data moves; i.e., the
351 // destination of region n is simply the start of region n. The argument beg
352 // must be region-aligned; end need not be.
353 void summarize_dense_prefix(HeapWord* beg, HeapWord* end);
354
355 HeapWord* summarize_split_space(size_t src_region, SplitInfo& split_info,
356 HeapWord* destination, HeapWord* target_end,
357 HeapWord** target_next);
358 bool summarize(SplitInfo& split_info,
359 HeapWord* source_beg, HeapWord* source_end,
360 HeapWord** source_next,
361 HeapWord* target_beg, HeapWord* target_end,
362 HeapWord** target_next);
363
364 void clear();
365 void clear_range(size_t beg_region, size_t end_region);
366 void clear_range(HeapWord* beg, HeapWord* end) {
367 clear_range(addr_to_region_idx(beg), addr_to_region_idx(end));
368 }
369
370 // Return the number of words between addr and the start of the region
371 // containing addr.
372 inline size_t region_offset(const HeapWord* addr) const;
373
374 // Convert addresses to/from a region index or region pointer.
375 inline size_t addr_to_region_idx(const HeapWord* addr) const;
376 inline RegionData* addr_to_region_ptr(const HeapWord* addr) const;
377 inline HeapWord* region_to_addr(size_t region) const;
378 inline HeapWord* region_to_addr(size_t region, size_t offset) const;
379 inline HeapWord* region_to_addr(const RegionData* region) const;
380
381 inline HeapWord* region_align_down(HeapWord* addr) const;
382 inline HeapWord* region_align_up(HeapWord* addr) const;
383 inline bool is_region_aligned(HeapWord* addr) const;
384
385 // Return the address one past the end of the partial object.
386 HeapWord* partial_obj_end(size_t region_idx) const;
387
388 // Return the new location of the object p after the
389 // the compaction.
390 HeapWord* calc_new_pointer(HeapWord* addr);
391
392 HeapWord* calc_new_pointer(oop p) {
393 return calc_new_pointer((HeapWord*) p);
394 }
395
396 // Return the updated address for the given klass
397 klassOop calc_new_klass(klassOop);
398
399 #ifdef ASSERT
400 void verify_clear(const PSVirtualSpace* vspace);
401 void verify_clear();
402 #endif // #ifdef ASSERT
403
404 private:
405 bool initialize_region_data(size_t region_size);
406 PSVirtualSpace* create_vspace(size_t count, size_t element_size);
407
408 private:
409 HeapWord* _region_start;
410 #ifdef ASSERT
411 HeapWord* _region_end;
412 #endif // #ifdef ASSERT
413
414 PSVirtualSpace* _region_vspace;
415 RegionData* _region_data;
416 size_t _region_count;
417 };
418
419 inline uint
420 ParallelCompactData::RegionData::destination_count_raw() const
421 {
422 return _dc_and_los & dc_mask;
423 }
424
425 inline uint
426 ParallelCompactData::RegionData::destination_count() const
427 {
428 return destination_count_raw() >> dc_shift;
429 }
430
431 inline void
432 ParallelCompactData::RegionData::set_destination_count(uint count)
433 {
434 assert(count <= (dc_completed >> dc_shift), "count too large");
435 const region_sz_t live_sz = (region_sz_t) live_obj_size();
436 _dc_and_los = (count << dc_shift) | live_sz;
437 }
438
439 inline void ParallelCompactData::RegionData::set_live_obj_size(size_t words)
440 {
441 assert(words <= los_mask, "would overflow");
442 _dc_and_los = destination_count_raw() | (region_sz_t)words;
443 }
444
445 inline void ParallelCompactData::RegionData::decrement_destination_count()
446 {
447 assert(_dc_and_los < dc_claimed, "already claimed");
448 assert(_dc_and_los >= dc_one, "count would go negative");
449 Atomic::add((int)dc_mask, (volatile int*)&_dc_and_los);
450 }
451
452 inline HeapWord* ParallelCompactData::RegionData::data_location() const
453 {
454 DEBUG_ONLY(return _data_location;)
455 NOT_DEBUG(return NULL;)
456 }
457
458 inline HeapWord* ParallelCompactData::RegionData::highest_ref() const
459 {
460 DEBUG_ONLY(return _highest_ref;)
461 NOT_DEBUG(return NULL;)
462 }
463
464 inline void ParallelCompactData::RegionData::set_data_location(HeapWord* addr)
465 {
466 DEBUG_ONLY(_data_location = addr;)
467 }
468
469 inline void ParallelCompactData::RegionData::set_completed()
470 {
471 assert(claimed(), "must be claimed first");
472 _dc_and_los = dc_completed | (region_sz_t) live_obj_size();
473 }
474
475 // MT-unsafe claiming of a region. Should only be used during single threaded
476 // execution.
477 inline bool ParallelCompactData::RegionData::claim_unsafe()
478 {
479 if (available()) {
480 _dc_and_los |= dc_claimed;
481 return true;
482 }
483 return false;
484 }
485
486 inline void ParallelCompactData::RegionData::add_live_obj(size_t words)
487 {
488 assert(words <= (size_t)los_mask - live_obj_size(), "overflow");
489 Atomic::add((int) words, (volatile int*) &_dc_and_los);
490 }
491
492 inline void ParallelCompactData::RegionData::set_highest_ref(HeapWord* addr)
493 {
494 #ifdef ASSERT
495 HeapWord* tmp = _highest_ref;
496 while (addr > tmp) {
497 tmp = (HeapWord*)Atomic::cmpxchg_ptr(addr, &_highest_ref, tmp);
498 }
499 #endif // #ifdef ASSERT
500 }
501
502 inline bool ParallelCompactData::RegionData::claim()
503 {
504 const int los = (int) live_obj_size();
505 const int old = Atomic::cmpxchg(dc_claimed | los,
506 (volatile int*) &_dc_and_los, los);
507 return old == los;
508 }
509
510 inline ParallelCompactData::RegionData*
511 ParallelCompactData::region(size_t region_idx) const
512 {
513 assert(region_idx <= region_count(), "bad arg");
514 return _region_data + region_idx;
515 }
516
517 inline size_t
518 ParallelCompactData::region(const RegionData* const region_ptr) const
519 {
520 assert(region_ptr >= _region_data, "bad arg");
521 assert(region_ptr <= _region_data + region_count(), "bad arg");
522 return pointer_delta(region_ptr, _region_data, sizeof(RegionData));
523 }
524
525 inline size_t
526 ParallelCompactData::region_offset(const HeapWord* addr) const
527 {
528 assert(addr >= _region_start, "bad addr");
529 assert(addr <= _region_end, "bad addr");
530 return (size_t(addr) & RegionAddrOffsetMask) >> LogHeapWordSize;
531 }
532
533 inline size_t
534 ParallelCompactData::addr_to_region_idx(const HeapWord* addr) const
535 {
536 assert(addr >= _region_start, "bad addr");
537 assert(addr <= _region_end, "bad addr");
538 return pointer_delta(addr, _region_start) >> Log2RegionSize;
539 }
540
541 inline ParallelCompactData::RegionData*
542 ParallelCompactData::addr_to_region_ptr(const HeapWord* addr) const
543 {
544 return region(addr_to_region_idx(addr));
545 }
546
547 inline HeapWord*
548 ParallelCompactData::region_to_addr(size_t region) const
549 {
550 assert(region <= _region_count, "region out of range");
551 return _region_start + (region << Log2RegionSize);
552 }
553
554 inline HeapWord*
555 ParallelCompactData::region_to_addr(const RegionData* region) const
556 {
557 return region_to_addr(pointer_delta(region, _region_data,
558 sizeof(RegionData)));
559 }
560
561 inline HeapWord*
562 ParallelCompactData::region_to_addr(size_t region, size_t offset) const
563 {
564 assert(region <= _region_count, "region out of range");
565 assert(offset < RegionSize, "offset too big"); // This may be too strict.
566 return region_to_addr(region) + offset;
567 }
568
569 inline HeapWord*
570 ParallelCompactData::region_align_down(HeapWord* addr) const
571 {
572 assert(addr >= _region_start, "bad addr");
573 assert(addr < _region_end + RegionSize, "bad addr");
574 return (HeapWord*)(size_t(addr) & RegionAddrMask);
575 }
576
577 inline HeapWord*
578 ParallelCompactData::region_align_up(HeapWord* addr) const
579 {
580 assert(addr >= _region_start, "bad addr");
581 assert(addr <= _region_end, "bad addr");
582 return region_align_down(addr + RegionSizeOffsetMask);
583 }
584
585 inline bool
586 ParallelCompactData::is_region_aligned(HeapWord* addr) const
587 {
588 return region_offset(addr) == 0;
589 }
590
591 // Abstract closure for use with ParMarkBitMap::iterate(), which will invoke the
592 // do_addr() method.
593 //
594 // The closure is initialized with the number of heap words to process
595 // (words_remaining()), and becomes 'full' when it reaches 0. The do_addr()
596 // methods in subclasses should update the total as words are processed. Since
597 // only one subclass actually uses this mechanism to terminate iteration, the
598 // default initial value is > 0. The implementation is here and not in the
599 // single subclass that uses it to avoid making is_full() virtual, and thus
600 // adding a virtual call per live object.
601
602 class ParMarkBitMapClosure: public StackObj {
603 public:
604 typedef ParMarkBitMap::idx_t idx_t;
605 typedef ParMarkBitMap::IterationStatus IterationStatus;
606
607 public:
608 inline ParMarkBitMapClosure(ParMarkBitMap* mbm, ParCompactionManager* cm,
609 size_t words = max_uintx);
610
611 inline ParCompactionManager* compaction_manager() const;
612 inline ParMarkBitMap* bitmap() const;
613 inline size_t words_remaining() const;
614 inline bool is_full() const;
615 inline HeapWord* source() const;
616
617 inline void set_source(HeapWord* addr);
618
619 virtual IterationStatus do_addr(HeapWord* addr, size_t words) = 0;
620
621 protected:
622 inline void decrement_words_remaining(size_t words);
623
624 private:
625 ParMarkBitMap* const _bitmap;
626 ParCompactionManager* const _compaction_manager;
627 DEBUG_ONLY(const size_t _initial_words_remaining;) // Useful in debugger.
628 size_t _words_remaining; // Words left to copy.
629
630 protected:
631 HeapWord* _source; // Next addr that would be read.
632 };
633
634 inline
635 ParMarkBitMapClosure::ParMarkBitMapClosure(ParMarkBitMap* bitmap,
636 ParCompactionManager* cm,
637 size_t words):
638 _bitmap(bitmap), _compaction_manager(cm)
639 #ifdef ASSERT
640 , _initial_words_remaining(words)
641 #endif
642 {
643 _words_remaining = words;
644 _source = NULL;
645 }
646
647 inline ParCompactionManager* ParMarkBitMapClosure::compaction_manager() const {
648 return _compaction_manager;
649 }
650
651 inline ParMarkBitMap* ParMarkBitMapClosure::bitmap() const {
652 return _bitmap;
653 }
654
655 inline size_t ParMarkBitMapClosure::words_remaining() const {
656 return _words_remaining;
657 }
658
659 inline bool ParMarkBitMapClosure::is_full() const {
660 return words_remaining() == 0;
661 }
662
663 inline HeapWord* ParMarkBitMapClosure::source() const {
664 return _source;
665 }
666
667 inline void ParMarkBitMapClosure::set_source(HeapWord* addr) {
668 _source = addr;
669 }
670
671 inline void ParMarkBitMapClosure::decrement_words_remaining(size_t words) {
672 assert(_words_remaining >= words, "processed too many words");
673 _words_remaining -= words;
674 }
675
676 // The UseParallelOldGC collector is a stop-the-world garbage collector that
677 // does parts of the collection using parallel threads. The collection includes
678 // the tenured generation and the young generation. The permanent generation is
679 // collected at the same time as the other two generations but the permanent
680 // generation is collect by a single GC thread. The permanent generation is
681 // collected serially because of the requirement that during the processing of a
682 // klass AAA, any objects reference by AAA must already have been processed.
683 // This requirement is enforced by a left (lower address) to right (higher
684 // address) sliding compaction.
685 //
686 // There are four phases of the collection.
687 //
688 // - marking phase
689 // - summary phase
690 // - compacting phase
691 // - clean up phase
692 //
693 // Roughly speaking these phases correspond, respectively, to
694 // - mark all the live objects
695 // - calculate the destination of each object at the end of the collection
696 // - move the objects to their destination
697 // - update some references and reinitialize some variables
698 //
699 // These three phases are invoked in PSParallelCompact::invoke_no_policy(). The
700 // marking phase is implemented in PSParallelCompact::marking_phase() and does a
701 // complete marking of the heap. The summary phase is implemented in
702 // PSParallelCompact::summary_phase(). The move and update phase is implemented
703 // in PSParallelCompact::compact().
704 //
705 // A space that is being collected is divided into regions and with each region
706 // is associated an object of type ParallelCompactData. Each region is of a
707 // fixed size and typically will contain more than 1 object and may have parts
708 // of objects at the front and back of the region.
709 //
710 // region -----+---------------------+----------
711 // objects covered [ AAA )[ BBB )[ CCC )[ DDD )
712 //
713 // The marking phase does a complete marking of all live objects in the heap.
714 // The marking also compiles the size of the data for all live objects covered
715 // by the region. This size includes the part of any live object spanning onto
716 // the region (part of AAA if it is live) from the front, all live objects
717 // contained in the region (BBB and/or CCC if they are live), and the part of
718 // any live objects covered by the region that extends off the region (part of
719 // DDD if it is live). The marking phase uses multiple GC threads and marking
720 // is done in a bit array of type ParMarkBitMap. The marking of the bit map is
721 // done atomically as is the accumulation of the size of the live objects
722 // covered by a region.
723 //
724 // The summary phase calculates the total live data to the left of each region
725 // XXX. Based on that total and the bottom of the space, it can calculate the
726 // starting location of the live data in XXX. The summary phase calculates for
727 // each region XXX quantites such as
728 //
729 // - the amount of live data at the beginning of a region from an object
730 // entering the region.
731 // - the location of the first live data on the region
732 // - a count of the number of regions receiving live data from XXX.
733 //
734 // See ParallelCompactData for precise details. The summary phase also
735 // calculates the dense prefix for the compaction. The dense prefix is a
736 // portion at the beginning of the space that is not moved. The objects in the
737 // dense prefix do need to have their object references updated. See method
738 // summarize_dense_prefix().
739 //
740 // The summary phase is done using 1 GC thread.
741 //
742 // The compaction phase moves objects to their new location and updates all
743 // references in the object.
744 //
745 // A current exception is that objects that cross a region boundary are moved
746 // but do not have their references updated. References are not updated because
747 // it cannot easily be determined if the klass pointer KKK for the object AAA
748 // has been updated. KKK likely resides in a region to the left of the region
749 // containing AAA. These AAA's have there references updated at the end in a
750 // clean up phase. See the method PSParallelCompact::update_deferred_objects().
751 // An alternate strategy is being investigated for this deferral of updating.
752 //
753 // Compaction is done on a region basis. A region that is ready to be filled is
754 // put on a ready list and GC threads take region off the list and fill them. A
755 // region is ready to be filled if it empty of live objects. Such a region may
756 // have been initially empty (only contained dead objects) or may have had all
757 // its live objects copied out already. A region that compacts into itself is
758 // also ready for filling. The ready list is initially filled with empty
759 // regions and regions compacting into themselves. There is always at least 1
760 // region that can be put on the ready list. The regions are atomically added
761 // and removed from the ready list.
762
763 class PSParallelCompact : AllStatic {
764 public:
765 // Convenient access to type names.
766 typedef ParMarkBitMap::idx_t idx_t;
767 typedef ParallelCompactData::RegionData RegionData;
768
769 typedef enum {
770 perm_space_id, old_space_id, eden_space_id,
771 from_space_id, to_space_id, last_space_id
772 } SpaceId;
773
774 public:
775 // Inline closure decls
776 //
777 class IsAliveClosure: public BoolObjectClosure {
778 public:
779 virtual void do_object(oop p);
780 virtual bool do_object_b(oop p);
781 };
782
783 class KeepAliveClosure: public OopClosure {
784 private:
785 ParCompactionManager* _compaction_manager;
786 protected:
787 template <class T> inline void do_oop_work(T* p);
788 public:
789 KeepAliveClosure(ParCompactionManager* cm) : _compaction_manager(cm) { }
790 virtual void do_oop(oop* p);
791 virtual void do_oop(narrowOop* p);
792 };
793
794 // Current unused
795 class FollowRootClosure: public OopsInGenClosure {
796 private:
797 ParCompactionManager* _compaction_manager;
798 public:
799 FollowRootClosure(ParCompactionManager* cm) : _compaction_manager(cm) { }
800 virtual void do_oop(oop* p);
801 virtual void do_oop(narrowOop* p);
802 };
803
804 class FollowStackClosure: public VoidClosure {
805 private:
806 ParCompactionManager* _compaction_manager;
807 public:
808 FollowStackClosure(ParCompactionManager* cm) : _compaction_manager(cm) { }
809 virtual void do_void();
810 };
811
812 class AdjustPointerClosure: public OopsInGenClosure {
813 private:
814 bool _is_root;
815 public:
816 AdjustPointerClosure(bool is_root) : _is_root(is_root) { }
817 virtual void do_oop(oop* p);
818 virtual void do_oop(narrowOop* p);
819 // do not walk from thread stacks to the code cache on this phase
820 virtual void do_code_blob(CodeBlob* cb) const { }
821 };
822
823 // Closure for verifying update of pointers. Does not
824 // have any side effects.
825 class VerifyUpdateClosure: public ParMarkBitMapClosure {
826 const MutableSpace* _space; // Is this ever used?
827
828 public:
829 VerifyUpdateClosure(ParCompactionManager* cm, const MutableSpace* sp) :
830 ParMarkBitMapClosure(PSParallelCompact::mark_bitmap(), cm), _space(sp)
831 { }
832
833 virtual IterationStatus do_addr(HeapWord* addr, size_t words);
834
835 const MutableSpace* space() { return _space; }
836 };
837
838 // Closure for updating objects altered for debug checking
839 class ResetObjectsClosure: public ParMarkBitMapClosure {
840 public:
841 ResetObjectsClosure(ParCompactionManager* cm):
842 ParMarkBitMapClosure(PSParallelCompact::mark_bitmap(), cm)
843 { }
844
845 virtual IterationStatus do_addr(HeapWord* addr, size_t words);
846 };
847
848 friend class KeepAliveClosure;
849 friend class FollowStackClosure;
850 friend class AdjustPointerClosure;
851 friend class FollowRootClosure;
852 friend class instanceKlassKlass;
853 friend class RefProcTaskProxy;
854
855 private:
856 static elapsedTimer _accumulated_time;
857 static unsigned int _total_invocations;
858 static unsigned int _maximum_compaction_gc_num;
859 static jlong _time_of_last_gc; // ms
860 static CollectorCounters* _counters;
861 static ParMarkBitMap _mark_bitmap;
862 static ParallelCompactData _summary_data;
863 static IsAliveClosure _is_alive_closure;
864 static SpaceInfo _space_info[last_space_id];
865 static bool _print_phases;
866 static AdjustPointerClosure _adjust_root_pointer_closure;
867 static AdjustPointerClosure _adjust_pointer_closure;
868
869 // Reference processing (used in ...follow_contents)
870 static ReferenceProcessor* _ref_processor;
871
872 // Updated location of intArrayKlassObj.
873 static klassOop _updated_int_array_klass_obj;
874
875 // Values computed at initialization and used by dead_wood_limiter().
876 static double _dwl_mean;
877 static double _dwl_std_dev;
878 static double _dwl_first_term;
879 static double _dwl_adjustment;
880 #ifdef ASSERT
881 static bool _dwl_initialized;
882 #endif // #ifdef ASSERT
883
884 private:
885 // Closure accessors
886 static OopClosure* adjust_pointer_closure() { return (OopClosure*)&_adjust_pointer_closure; }
887 static OopClosure* adjust_root_pointer_closure() { return (OopClosure*)&_adjust_root_pointer_closure; }
888 static BoolObjectClosure* is_alive_closure() { return (BoolObjectClosure*)&_is_alive_closure; }
889
890 static void initialize_space_info();
891
892 // Return true if details about individual phases should be printed.
893 static inline bool print_phases();
894
895 // Clear the marking bitmap and summary data that cover the specified space.
896 static void clear_data_covering_space(SpaceId id);
897
898 static void pre_compact(PreGCValues* pre_gc_values);
899 static void post_compact();
900
901 // Mark live objects
902 static void marking_phase(ParCompactionManager* cm,
903 bool maximum_heap_compaction);
904 static void follow_weak_klass_links();
905 static void follow_mdo_weak_refs();
906
907 template <class T> static inline void adjust_pointer(T* p, bool is_root);
908 static void adjust_root_pointer(oop* p) { adjust_pointer(p, true); }
909
910 template <class T>
911 static inline void follow_root(ParCompactionManager* cm, T* p);
912
913 // Compute the dense prefix for the designated space. This is an experimental
914 // implementation currently not used in production.
915 static HeapWord* compute_dense_prefix_via_density(const SpaceId id,
916 bool maximum_compaction);
917
918 // Methods used to compute the dense prefix.
919
920 // Compute the value of the normal distribution at x = density. The mean and
921 // standard deviation are values saved by initialize_dead_wood_limiter().
922 static inline double normal_distribution(double density);
923
924 // Initialize the static vars used by dead_wood_limiter().
925 static void initialize_dead_wood_limiter();
926
927 // Return the percentage of space that can be treated as "dead wood" (i.e.,
928 // not reclaimed).
929 static double dead_wood_limiter(double density, size_t min_percent);
930
931 // Find the first (left-most) region in the range [beg, end) that has at least
932 // dead_words of dead space to the left. The argument beg must be the first
933 // region in the space that is not completely live.
934 static RegionData* dead_wood_limit_region(const RegionData* beg,
935 const RegionData* end,
936 size_t dead_words);
937
938 // Return a pointer to the first region in the range [beg, end) that is not
939 // completely full.
940 static RegionData* first_dead_space_region(const RegionData* beg,
941 const RegionData* end);
942
943 // Return a value indicating the benefit or 'yield' if the compacted region
944 // were to start (or equivalently if the dense prefix were to end) at the
945 // candidate region. Higher values are better.
946 //
947 // The value is based on the amount of space reclaimed vs. the costs of (a)
948 // updating references in the dense prefix plus (b) copying objects and
949 // updating references in the compacted region.
950 static inline double reclaimed_ratio(const RegionData* const candidate,
951 HeapWord* const bottom,
952 HeapWord* const top,
953 HeapWord* const new_top);
954
955 // Compute the dense prefix for the designated space.
956 static HeapWord* compute_dense_prefix(const SpaceId id,
957 bool maximum_compaction);
958
959 // Return true if dead space crosses onto the specified Region; bit must be
960 // the bit index corresponding to the first word of the Region.
961 static inline bool dead_space_crosses_boundary(const RegionData* region,
962 idx_t bit);
963
964 // Summary phase utility routine to fill dead space (if any) at the dense
965 // prefix boundary. Should only be called if the the dense prefix is
966 // non-empty.
967 static void fill_dense_prefix_end(SpaceId id);
968
969 // Clear the summary data source_region field for the specified addresses.
970 static void clear_source_region(HeapWord* beg_addr, HeapWord* end_addr);
971
972 #ifndef PRODUCT
973 // Routines to provoke splitting a young gen space (ParallelOldGCSplitALot).
974
975 // Fill the region [start, start + words) with live object(s). Only usable
976 // for the old and permanent generations.
977 static void fill_with_live_objects(SpaceId id, HeapWord* const start,
978 size_t words);
979 // Include the new objects in the summary data.
980 static void summarize_new_objects(SpaceId id, HeapWord* start);
981
982 // Add live objects to a survivor space since it's rare that both survivors
983 // are non-empty.
984 static void provoke_split_fill_survivor(SpaceId id);
985
986 // Add live objects and/or choose the dense prefix to provoke splitting.
987 static void provoke_split(bool & maximum_compaction);
988 #endif
989
990 static void summarize_spaces_quick();
991 static void summarize_space(SpaceId id, bool maximum_compaction);
992 static void summary_phase(ParCompactionManager* cm, bool maximum_compaction);
993
994 // Adjust addresses in roots. Does not adjust addresses in heap.
995 static void adjust_roots();
996
997 // Serial code executed in preparation for the compaction phase.
998 static void compact_prologue();
999
1000 // Move objects to new locations.
1001 static void compact_perm(ParCompactionManager* cm);
1002 static void compact();
1003
1004 // Add available regions to the stack and draining tasks to the task queue.
1005 static void enqueue_region_draining_tasks(GCTaskQueue* q,
1006 uint parallel_gc_threads);
1007
1008 // Add dense prefix update tasks to the task queue.
1009 static void enqueue_dense_prefix_tasks(GCTaskQueue* q,
1010 uint parallel_gc_threads);
1011
1012 // Add region stealing tasks to the task queue.
1013 static void enqueue_region_stealing_tasks(
1014 GCTaskQueue* q,
1015 ParallelTaskTerminator* terminator_ptr,
1016 uint parallel_gc_threads);
1017
1018 // For debugging only - compacts the old gen serially
1019 static void compact_serial(ParCompactionManager* cm);
1020
1021 // If objects are left in eden after a collection, try to move the boundary
1022 // and absorb them into the old gen. Returns true if eden was emptied.
1023 static bool absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy,
1024 PSYoungGen* young_gen,
1025 PSOldGen* old_gen);
1026
1027 // Reset time since last full gc
1028 static void reset_millis_since_last_gc();
1029
1030 protected:
1031 #ifdef VALIDATE_MARK_SWEEP
1032 static GrowableArray<void*>* _root_refs_stack;
1033 static GrowableArray<oop> * _live_oops;
1034 static GrowableArray<oop> * _live_oops_moved_to;
1035 static GrowableArray<size_t>* _live_oops_size;
1036 static size_t _live_oops_index;
1037 static size_t _live_oops_index_at_perm;
1038 static GrowableArray<void*>* _other_refs_stack;
1039 static GrowableArray<void*>* _adjusted_pointers;
1040 static bool _pointer_tracking;
1041 static bool _root_tracking;
1042
1043 // The following arrays are saved since the time of the last GC and
1044 // assist in tracking down problems where someone has done an errant
1045 // store into the heap, usually to an oop that wasn't properly
1046 // handleized across a GC. If we crash or otherwise fail before the
1047 // next GC, we can query these arrays to find out the object we had
1048 // intended to do the store to (assuming it is still alive) and the
1049 // offset within that object. Covered under RecordMarkSweepCompaction.
1050 static GrowableArray<HeapWord*> * _cur_gc_live_oops;
1051 static GrowableArray<HeapWord*> * _cur_gc_live_oops_moved_to;
1052 static GrowableArray<size_t>* _cur_gc_live_oops_size;
1053 static GrowableArray<HeapWord*> * _last_gc_live_oops;
1054 static GrowableArray<HeapWord*> * _last_gc_live_oops_moved_to;
1055 static GrowableArray<size_t>* _last_gc_live_oops_size;
1056 #endif
1057
1058 public:
1059 class MarkAndPushClosure: public OopClosure {
1060 private:
1061 ParCompactionManager* _compaction_manager;
1062 public:
1063 MarkAndPushClosure(ParCompactionManager* cm) : _compaction_manager(cm) { }
1064 virtual void do_oop(oop* p);
1065 virtual void do_oop(narrowOop* p);
1066 };
1067
1068 PSParallelCompact();
1069
1070 // Convenient accessor for Universe::heap().
1071 static ParallelScavengeHeap* gc_heap() {
1072 return (ParallelScavengeHeap*)Universe::heap();
1073 }
1074
1075 static void invoke(bool maximum_heap_compaction);
1076 static void invoke_no_policy(bool maximum_heap_compaction);
1077
1078 static void post_initialize();
1079 // Perform initialization for PSParallelCompact that requires
1080 // allocations. This should be called during the VM initialization
1081 // at a pointer where it would be appropriate to return a JNI_ENOMEM
1082 // in the event of a failure.
1083 static bool initialize();
1084
1085 // Public accessors
1086 static elapsedTimer* accumulated_time() { return &_accumulated_time; }
1087 static unsigned int total_invocations() { return _total_invocations; }
1088 static CollectorCounters* counters() { return _counters; }
1089
1090 // Used to add tasks
1091 static GCTaskManager* const gc_task_manager();
1092 static klassOop updated_int_array_klass_obj() {
1093 return _updated_int_array_klass_obj;
1094 }
1095
1096 // Marking support
1097 static inline bool mark_obj(oop obj);
1098 // Check mark and maybe push on marking stack
1099 template <class T> static inline void mark_and_push(ParCompactionManager* cm,
1100 T* p);
1101
1102 // Compaction support.
1103 // Return true if p is in the range [beg_addr, end_addr).
1104 static inline bool is_in(HeapWord* p, HeapWord* beg_addr, HeapWord* end_addr);
1105 static inline bool is_in(oop* p, HeapWord* beg_addr, HeapWord* end_addr);
1106
1107 // Convenience wrappers for per-space data kept in _space_info.
1108 static inline MutableSpace* space(SpaceId space_id);
1109 static inline HeapWord* new_top(SpaceId space_id);
1110 static inline HeapWord* dense_prefix(SpaceId space_id);
1111 static inline ObjectStartArray* start_array(SpaceId space_id);
1112
1113 // Return true if the klass should be updated.
1114 static inline bool should_update_klass(klassOop k);
1115
1116 // Move and update the live objects in the specified space.
1117 static void move_and_update(ParCompactionManager* cm, SpaceId space_id);
1118
1119 // Process the end of the given region range in the dense prefix.
1120 // This includes saving any object not updated.
1121 static void dense_prefix_regions_epilogue(ParCompactionManager* cm,
1122 size_t region_start_index,
1123 size_t region_end_index,
1124 idx_t exiting_object_offset,
1125 idx_t region_offset_start,
1126 idx_t region_offset_end);
1127
1128 // Update a region in the dense prefix. For each live object
1129 // in the region, update it's interior references. For each
1130 // dead object, fill it with deadwood. Dead space at the end
1131 // of a region range will be filled to the start of the next
1132 // live object regardless of the region_index_end. None of the
1133 // objects in the dense prefix move and dead space is dead
1134 // (holds only dead objects that don't need any processing), so
1135 // dead space can be filled in any order.
1136 static void update_and_deadwood_in_dense_prefix(ParCompactionManager* cm,
1137 SpaceId space_id,
1138 size_t region_index_start,
1139 size_t region_index_end);
1140
1141 // Return the address of the count + 1st live word in the range [beg, end).
1142 static HeapWord* skip_live_words(HeapWord* beg, HeapWord* end, size_t count);
1143
1144 // Return the address of the word to be copied to dest_addr, which must be
1145 // aligned to a region boundary.
1146 static HeapWord* first_src_addr(HeapWord* const dest_addr,
1147 SpaceId src_space_id,
1148 size_t src_region_idx);
1149
1150 // Determine the next source region, set closure.source() to the start of the
1151 // new region return the region index. Parameter end_addr is the address one
1152 // beyond the end of source range just processed. If necessary, switch to a
1153 // new source space and set src_space_id (in-out parameter) and src_space_top
1154 // (out parameter) accordingly.
1155 static size_t next_src_region(MoveAndUpdateClosure& closure,
1156 SpaceId& src_space_id,
1157 HeapWord*& src_space_top,
1158 HeapWord* end_addr);
1159
1160 // Decrement the destination count for each non-empty source region in the
1161 // range [beg_region, region(region_align_up(end_addr))). If the destination
1162 // count for a region goes to 0 and it needs to be filled, enqueue it.
1163 static void decrement_destination_counts(ParCompactionManager* cm,
1164 SpaceId src_space_id,
1165 size_t beg_region,
1166 HeapWord* end_addr);
1167
1168 // Fill a region, copying objects from one or more source regions.
1169 static void fill_region(ParCompactionManager* cm, size_t region_idx);
1170 static void fill_and_update_region(ParCompactionManager* cm, size_t region) {
1171 fill_region(cm, region);
1172 }
1173
1174 // Update the deferred objects in the space.
1175 static void update_deferred_objects(ParCompactionManager* cm, SpaceId id);
1176
1177 // Mark pointer and follow contents.
1178 template <class T>
1179 static inline void mark_and_follow(ParCompactionManager* cm, T* p);
1180
1181 static ParMarkBitMap* mark_bitmap() { return &_mark_bitmap; }
1182 static ParallelCompactData& summary_data() { return _summary_data; }
1183
1184 static inline void adjust_pointer(oop* p) { adjust_pointer(p, false); }
1185 static inline void adjust_pointer(narrowOop* p) { adjust_pointer(p, false); }
1186
1187 template <class T>
1188 static inline void adjust_pointer(T* p,
1189 HeapWord* beg_addr,
1190 HeapWord* end_addr);
1191
1192 // Reference Processing
1193 static ReferenceProcessor* const ref_processor() { return _ref_processor; }
1194
1195 // Return the SpaceId for the given address.
1196 static SpaceId space_id(HeapWord* addr);
1197
1198 // Time since last full gc (in milliseconds).
1199 static jlong millis_since_last_gc();
1200
1201 #ifdef VALIDATE_MARK_SWEEP
1202 static void track_adjusted_pointer(void* p, bool isroot);
1203 static void check_adjust_pointer(void* p);
1204 static void track_interior_pointers(oop obj);
1205 static void check_interior_pointers();
1206
1207 static void reset_live_oop_tracking(bool at_perm);
1208 static void register_live_oop(oop p, size_t size);
1209 static void validate_live_oop(oop p, size_t size);
1210 static void live_oop_moved_to(HeapWord* q, size_t size, HeapWord* compaction_top);
1211 static void compaction_complete();
1212
1213 // Querying operation of RecordMarkSweepCompaction results.
1214 // Finds and prints the current base oop and offset for a word
1215 // within an oop that was live during the last GC. Helpful for
1216 // tracking down heap stomps.
1217 static void print_new_location_of_heap_address(HeapWord* q);
1218 #endif // #ifdef VALIDATE_MARK_SWEEP
1219
1220 // Call backs for class unloading
1221 // Update subklass/sibling/implementor links at end of marking.
1222 static void revisit_weak_klass_link(ParCompactionManager* cm, Klass* k);
1223
1224 // Clear unmarked oops in MDOs at the end of marking.
1225 static void revisit_mdo(ParCompactionManager* cm, DataLayout* p);
1226
1227 #ifndef PRODUCT
1228 // Debugging support.
1229 static const char* space_names[last_space_id];
1230 static void print_region_ranges();
1231 static void print_dense_prefix_stats(const char* const algorithm,
1232 const SpaceId id,
1233 const bool maximum_compaction,
1234 HeapWord* const addr);
1235 static void summary_phase_msg(SpaceId dst_space_id,
1236 HeapWord* dst_beg, HeapWord* dst_end,
1237 SpaceId src_space_id,
1238 HeapWord* src_beg, HeapWord* src_end);
1239 #endif // #ifndef PRODUCT
1240
1241 #ifdef ASSERT
1242 // Sanity check the new location of a word in the heap.
1243 static inline void check_new_location(HeapWord* old_addr, HeapWord* new_addr);
1244 // Verify that all the regions have been emptied.
1245 static void verify_complete(SpaceId space_id);
1246 #endif // #ifdef ASSERT
1247 };
1248
1249 inline bool PSParallelCompact::mark_obj(oop obj) {
1250 const int obj_size = obj->size();
1251 if (mark_bitmap()->mark_obj(obj, obj_size)) {
1252 _summary_data.add_obj(obj, obj_size);
1253 return true;
1254 } else {
1255 return false;
1256 }
1257 }
1258
1259 template <class T>
1260 inline void PSParallelCompact::follow_root(ParCompactionManager* cm, T* p) {
1261 assert(!Universe::heap()->is_in_reserved(p),
1262 "roots shouldn't be things within the heap");
1263 #ifdef VALIDATE_MARK_SWEEP
1264 if (ValidateMarkSweep) {
1265 guarantee(!_root_refs_stack->contains(p), "should only be in here once");
1266 _root_refs_stack->push(p);
1267 }
1268 #endif
1269 T heap_oop = oopDesc::load_heap_oop(p);
1270 if (!oopDesc::is_null(heap_oop)) {
1271 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
1272 if (mark_bitmap()->is_unmarked(obj)) {
1273 if (mark_obj(obj)) {
1274 obj->follow_contents(cm);
1275 }
1276 }
1277 }
1278 cm->follow_marking_stacks();
1279 }
1280
1281 template <class T>
1282 inline void PSParallelCompact::mark_and_follow(ParCompactionManager* cm,
1283 T* p) {
1284 T heap_oop = oopDesc::load_heap_oop(p);
1285 if (!oopDesc::is_null(heap_oop)) {
1286 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
1287 if (mark_bitmap()->is_unmarked(obj)) {
1288 if (mark_obj(obj)) {
1289 obj->follow_contents(cm);
1290 }
1291 }
1292 }
1293 }
1294
1295 template <class T>
1296 inline void PSParallelCompact::mark_and_push(ParCompactionManager* cm, T* p) {
1297 T heap_oop = oopDesc::load_heap_oop(p);
1298 if (!oopDesc::is_null(heap_oop)) {
1299 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
1300 if (mark_bitmap()->is_unmarked(obj) && mark_obj(obj)) {
1301 cm->push(obj);
1302 }
1303 }
1304 }
1305
1306 template <class T>
1307 inline void PSParallelCompact::adjust_pointer(T* p, bool isroot) {
1308 T heap_oop = oopDesc::load_heap_oop(p);
1309 if (!oopDesc::is_null(heap_oop)) {
1310 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
1311 oop new_obj = (oop)summary_data().calc_new_pointer(obj);
1312 assert(new_obj != NULL || // is forwarding ptr?
1313 obj->is_shared(), // never forwarded?
1314 "should be forwarded");
1315 // Just always do the update unconditionally?
1316 if (new_obj != NULL) {
1317 assert(Universe::heap()->is_in_reserved(new_obj),
1318 "should be in object space");
1319 oopDesc::encode_store_heap_oop_not_null(p, new_obj);
1320 }
1321 }
1322 VALIDATE_MARK_SWEEP_ONLY(track_adjusted_pointer(p, isroot));
1323 }
1324
1325 template <class T>
1326 inline void PSParallelCompact::KeepAliveClosure::do_oop_work(T* p) {
1327 #ifdef VALIDATE_MARK_SWEEP
1328 if (ValidateMarkSweep) {
1329 if (!Universe::heap()->is_in_reserved(p)) {
1330 _root_refs_stack->push(p);
1331 } else {
1332 _other_refs_stack->push(p);
1333 }
1334 }
1335 #endif
1336 mark_and_push(_compaction_manager, p);
1337 }
1338
1339 inline bool PSParallelCompact::print_phases() {
1340 return _print_phases;
1341 }
1342
1343 inline double PSParallelCompact::normal_distribution(double density) {
1344 assert(_dwl_initialized, "uninitialized");
1345 const double squared_term = (density - _dwl_mean) / _dwl_std_dev;
1346 return _dwl_first_term * exp(-0.5 * squared_term * squared_term);
1347 }
1348
1349 inline bool
1350 PSParallelCompact::dead_space_crosses_boundary(const RegionData* region,
1351 idx_t bit)
1352 {
1353 assert(bit > 0, "cannot call this for the first bit/region");
1354 assert(_summary_data.region_to_addr(region) == _mark_bitmap.bit_to_addr(bit),
1355 "sanity check");
1356
1357 // Dead space crosses the boundary if (1) a partial object does not extend
1358 // onto the region, (2) an object does not start at the beginning of the
1359 // region, and (3) an object does not end at the end of the prior region.
1360 return region->partial_obj_size() == 0 &&
1361 !_mark_bitmap.is_obj_beg(bit) &&
1362 !_mark_bitmap.is_obj_end(bit - 1);
1363 }
1364
1365 inline bool
1366 PSParallelCompact::is_in(HeapWord* p, HeapWord* beg_addr, HeapWord* end_addr) {
1367 return p >= beg_addr && p < end_addr;
1368 }
1369
1370 inline bool
1371 PSParallelCompact::is_in(oop* p, HeapWord* beg_addr, HeapWord* end_addr) {
1372 return is_in((HeapWord*)p, beg_addr, end_addr);
1373 }
1374
1375 inline MutableSpace* PSParallelCompact::space(SpaceId id) {
1376 assert(id < last_space_id, "id out of range");
1377 return _space_info[id].space();
1378 }
1379
1380 inline HeapWord* PSParallelCompact::new_top(SpaceId id) {
1381 assert(id < last_space_id, "id out of range");
1382 return _space_info[id].new_top();
1383 }
1384
1385 inline HeapWord* PSParallelCompact::dense_prefix(SpaceId id) {
1386 assert(id < last_space_id, "id out of range");
1387 return _space_info[id].dense_prefix();
1388 }
1389
1390 inline ObjectStartArray* PSParallelCompact::start_array(SpaceId id) {
1391 assert(id < last_space_id, "id out of range");
1392 return _space_info[id].start_array();
1393 }
1394
1395 inline bool PSParallelCompact::should_update_klass(klassOop k) {
1396 return ((HeapWord*) k) >= dense_prefix(perm_space_id);
1397 }
1398
1399 template <class T>
1400 inline void PSParallelCompact::adjust_pointer(T* p,
1401 HeapWord* beg_addr,
1402 HeapWord* end_addr) {
1403 if (is_in((HeapWord*)p, beg_addr, end_addr)) {
1404 adjust_pointer(p);
1405 }
1406 }
1407
1408 #ifdef ASSERT
1409 inline void
1410 PSParallelCompact::check_new_location(HeapWord* old_addr, HeapWord* new_addr)
1411 {
1412 assert(old_addr >= new_addr || space_id(old_addr) != space_id(new_addr),
1413 "must move left or to a different space");
1414 assert(is_object_aligned((intptr_t)old_addr) && is_object_aligned((intptr_t)new_addr),
1415 "checking alignment");
1416 }
1417 #endif // ASSERT
1418
1419 class MoveAndUpdateClosure: public ParMarkBitMapClosure {
1420 public:
1421 inline MoveAndUpdateClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm,
1422 ObjectStartArray* start_array,
1423 HeapWord* destination, size_t words);
1424
1425 // Accessors.
1426 HeapWord* destination() const { return _destination; }
1427
1428 // If the object will fit (size <= words_remaining()), copy it to the current
1429 // destination, update the interior oops and the start array and return either
1430 // full (if the closure is full) or incomplete. If the object will not fit,
1431 // return would_overflow.
1432 virtual IterationStatus do_addr(HeapWord* addr, size_t size);
1433
1434 // Copy enough words to fill this closure, starting at source(). Interior
1435 // oops and the start array are not updated. Return full.
1436 IterationStatus copy_until_full();
1437
1438 // Copy enough words to fill this closure or to the end of an object,
1439 // whichever is smaller, starting at source(). Interior oops and the start
1440 // array are not updated.
1441 void copy_partial_obj();
1442
1443 protected:
1444 // Update variables to indicate that word_count words were processed.
1445 inline void update_state(size_t word_count);
1446
1447 protected:
1448 ObjectStartArray* const _start_array;
1449 HeapWord* _destination; // Next addr to be written.
1450 };
1451
1452 inline
1453 MoveAndUpdateClosure::MoveAndUpdateClosure(ParMarkBitMap* bitmap,
1454 ParCompactionManager* cm,
1455 ObjectStartArray* start_array,
1456 HeapWord* destination,
1457 size_t words) :
1458 ParMarkBitMapClosure(bitmap, cm, words), _start_array(start_array)
1459 {
1460 _destination = destination;
1461 }
1462
1463 inline void MoveAndUpdateClosure::update_state(size_t words)
1464 {
1465 decrement_words_remaining(words);
1466 _source += words;
1467 _destination += words;
1468 }
1469
1470 class UpdateOnlyClosure: public ParMarkBitMapClosure {
1471 private:
1472 const PSParallelCompact::SpaceId _space_id;
1473 ObjectStartArray* const _start_array;
1474
1475 public:
1476 UpdateOnlyClosure(ParMarkBitMap* mbm,
1477 ParCompactionManager* cm,
1478 PSParallelCompact::SpaceId space_id);
1479
1480 // Update the object.
1481 virtual IterationStatus do_addr(HeapWord* addr, size_t words);
1482
1483 inline void do_addr(HeapWord* addr);
1484 };
1485
1486 inline void UpdateOnlyClosure::do_addr(HeapWord* addr)
1487 {
1488 _start_array->allocate_block(addr);
1489 oop(addr)->update_contents(compaction_manager());
1490 }
1491
1492 class FillClosure: public ParMarkBitMapClosure
1493 {
1494 public:
1495 FillClosure(ParCompactionManager* cm, PSParallelCompact::SpaceId space_id) :
1496 ParMarkBitMapClosure(PSParallelCompact::mark_bitmap(), cm),
1497 _start_array(PSParallelCompact::start_array(space_id))
1498 {
1499 assert(space_id == PSParallelCompact::perm_space_id ||
1500 space_id == PSParallelCompact::old_space_id,
1501 "cannot use FillClosure in the young gen");
1502 }
1503
1504 virtual IterationStatus do_addr(HeapWord* addr, size_t size) {
1505 CollectedHeap::fill_with_objects(addr, size);
1506 HeapWord* const end = addr + size;
1507 do {
1508 _start_array->allocate_block(addr);
1509 addr += oop(addr)->size();
1510 } while (addr < end);
1511 return ParMarkBitMap::incomplete;
1512 }
1513
1514 private:
1515 ObjectStartArray* const _start_array;
1516 };