1 /*
   2  * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_HPP
  26 #define SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_HPP
  27 
  28 #include "gc/cms/cmsOopClosures.hpp"
  29 #include "gc/cms/gSpaceCounters.hpp"
  30 #include "gc/cms/yieldingWorkgroup.hpp"
  31 #include "gc/shared/cardGeneration.hpp"
  32 #include "gc/shared/gcHeapSummary.hpp"
  33 #include "gc/shared/gcStats.hpp"
  34 #include "gc/shared/gcWhen.hpp"
  35 #include "gc/shared/generationCounters.hpp"
  36 #include "gc/shared/space.hpp"
  37 #include "gc/shared/taskqueue.hpp"
  38 #include "logging/log.hpp"
  39 #include "memory/iterator.hpp"
  40 #include "memory/virtualspace.hpp"
  41 #include "runtime/mutexLocker.hpp"
  42 #include "services/memoryService.hpp"
  43 #include "utilities/bitMap.hpp"
  44 #include "utilities/stack.hpp"
  45 
  46 // ConcurrentMarkSweepGeneration is in support of a concurrent
  47 // mark-sweep old generation in the Detlefs-Printezis--Boehm-Demers-Schenker
  48 // style. We assume, for now, that this generation is always the
  49 // seniormost generation and for simplicity
  50 // in the first implementation, that this generation is a single compactible
  51 // space. Neither of these restrictions appears essential, and will be
  52 // relaxed in the future when more time is available to implement the
  53 // greater generality (and there's a need for it).
  54 //
  55 // Concurrent mode failures are currently handled by
  56 // means of a sliding mark-compact.
  57 
  58 class AdaptiveSizePolicy;
  59 class CMSCollector;
  60 class CMSConcMarkingTask;
  61 class CMSGCAdaptivePolicyCounters;
  62 class CMSTracer;
  63 class ConcurrentGCTimer;
  64 class ConcurrentMarkSweepGeneration;
  65 class ConcurrentMarkSweepPolicy;
  66 class ConcurrentMarkSweepThread;
  67 class CompactibleFreeListSpace;
  68 class FreeChunk;
  69 class ParNewGeneration;
  70 class PromotionInfo;
  71 class ScanMarkedObjectsAgainCarefullyClosure;
  72 class TenuredGeneration;
  73 class SerialOldTracer;
  74 
  75 // A generic CMS bit map. It's the basis for both the CMS marking bit map
  76 // as well as for the mod union table (in each case only a subset of the
  77 // methods are used). This is essentially a wrapper around the BitMap class,
  78 // with one bit per (1<<_shifter) HeapWords. (i.e. for the marking bit map,
  79 // we have _shifter == 0. and for the mod union table we have
  80 // shifter == CardTable::card_shift - LogHeapWordSize.)
  81 // XXX 64-bit issues in BitMap?
  82 class CMSBitMap VALUE_OBJ_CLASS_SPEC {
  83   friend class VMStructs;
  84 
  85   HeapWord*    _bmStartWord;   // base address of range covered by map
  86   size_t       _bmWordSize;    // map size (in #HeapWords covered)
  87   const int    _shifter;       // shifts to convert HeapWord to bit position
  88   VirtualSpace _virtual_space; // underlying the bit map
  89   BitMapView   _bm;            // the bit map itself
  90   Mutex* const _lock;          // mutex protecting _bm;
  91 
  92  public:
  93   // constructor
  94   CMSBitMap(int shifter, int mutex_rank, const char* mutex_name);
  95 
  96   // allocates the actual storage for the map
  97   bool allocate(MemRegion mr);
  98   // field getter
  99   Mutex* lock() const { return _lock; }
 100   // locking verifier convenience function
 101   void assert_locked() const PRODUCT_RETURN;
 102 
 103   // inquiries
 104   HeapWord* startWord()   const { return _bmStartWord; }
 105   size_t    sizeInWords() const { return _bmWordSize;  }
 106   size_t    sizeInBits()  const { return _bm.size();   }
 107   // the following is one past the last word in space
 108   HeapWord* endWord()     const { return _bmStartWord + _bmWordSize; }
 109 
 110   // reading marks
 111   bool isMarked(HeapWord* addr) const;
 112   bool par_isMarked(HeapWord* addr) const; // do not lock checks
 113   bool isUnmarked(HeapWord* addr) const;
 114   bool isAllClear() const;
 115 
 116   // writing marks
 117   void mark(HeapWord* addr);
 118   // For marking by parallel GC threads;
 119   // returns true if we did, false if another thread did
 120   bool par_mark(HeapWord* addr);
 121 
 122   void mark_range(MemRegion mr);
 123   void par_mark_range(MemRegion mr);
 124   void mark_large_range(MemRegion mr);
 125   void par_mark_large_range(MemRegion mr);
 126   void par_clear(HeapWord* addr); // For unmarking by parallel GC threads.
 127   void clear_range(MemRegion mr);
 128   void par_clear_range(MemRegion mr);
 129   void clear_large_range(MemRegion mr);
 130   void par_clear_large_range(MemRegion mr);
 131   void clear_all();
 132   void clear_all_incrementally();  // Not yet implemented!!
 133 
 134   NOT_PRODUCT(
 135     // checks the memory region for validity
 136     void region_invariant(MemRegion mr);
 137   )
 138 
 139   // iteration
 140   void iterate(BitMapClosure* cl) {
 141     _bm.iterate(cl);
 142   }
 143   void iterate(BitMapClosure* cl, HeapWord* left, HeapWord* right);
 144   void dirty_range_iterate_clear(MemRegionClosure* cl);
 145   void dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl);
 146 
 147   // auxiliary support for iteration
 148   HeapWord* getNextMarkedWordAddress(HeapWord* addr) const;
 149   HeapWord* getNextMarkedWordAddress(HeapWord* start_addr,
 150                                             HeapWord* end_addr) const;
 151   HeapWord* getNextUnmarkedWordAddress(HeapWord* addr) const;
 152   HeapWord* getNextUnmarkedWordAddress(HeapWord* start_addr,
 153                                               HeapWord* end_addr) const;
 154   MemRegion getAndClearMarkedRegion(HeapWord* addr);
 155   MemRegion getAndClearMarkedRegion(HeapWord* start_addr,
 156                                            HeapWord* end_addr);
 157 
 158   // conversion utilities
 159   HeapWord* offsetToHeapWord(size_t offset) const;
 160   size_t    heapWordToOffset(HeapWord* addr) const;
 161   size_t    heapWordDiffToOffsetDiff(size_t diff) const;
 162 
 163   void print_on_error(outputStream* st, const char* prefix) const;
 164 
 165   // debugging
 166   // is this address range covered by the bit-map?
 167   NOT_PRODUCT(
 168     bool covers(MemRegion mr) const;
 169     bool covers(HeapWord* start, size_t size = 0) const;
 170   )
 171   void verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) PRODUCT_RETURN;
 172 };
 173 
 174 // Represents a marking stack used by the CMS collector.
 175 // Ideally this should be GrowableArray<> just like MSC's marking stack(s).
 176 class CMSMarkStack: public CHeapObj<mtGC>  {
 177   friend class CMSCollector;   // To get at expansion stats further below.
 178 
 179   VirtualSpace _virtual_space;  // Space for the stack
 180   oop*   _base;      // Bottom of stack
 181   size_t _index;     // One more than last occupied index
 182   size_t _capacity;  // Max #elements
 183   Mutex  _par_lock;  // An advisory lock used in case of parallel access
 184   NOT_PRODUCT(size_t _max_depth;)  // Max depth plumbed during run
 185 
 186  protected:
 187   size_t _hit_limit;      // We hit max stack size limit
 188   size_t _failed_double;  // We failed expansion before hitting limit
 189 
 190  public:
 191   CMSMarkStack():
 192     _par_lock(Mutex::event, "CMSMarkStack._par_lock", true,
 193               Monitor::_safepoint_check_never),
 194     _hit_limit(0),
 195     _failed_double(0) {}
 196 
 197   bool allocate(size_t size);
 198 
 199   size_t capacity() const { return _capacity; }
 200 
 201   oop pop() {
 202     if (!isEmpty()) {
 203       return _base[--_index] ;
 204     }
 205     return NULL;
 206   }
 207 
 208   bool push(oop ptr) {
 209     if (isFull()) {
 210       return false;
 211     } else {
 212       _base[_index++] = ptr;
 213       NOT_PRODUCT(_max_depth = MAX2(_max_depth, _index));
 214       return true;
 215     }
 216   }
 217 
 218   bool isEmpty() const { return _index == 0; }
 219   bool isFull()  const {
 220     assert(_index <= _capacity, "buffer overflow");
 221     return _index == _capacity;
 222   }
 223 
 224   size_t length() { return _index; }
 225 
 226   // "Parallel versions" of some of the above
 227   oop par_pop() {
 228     // lock and pop
 229     MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag);
 230     return pop();
 231   }
 232 
 233   bool par_push(oop ptr) {
 234     // lock and push
 235     MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag);
 236     return push(ptr);
 237   }
 238 
 239   // Forcibly reset the stack, losing all of its contents.
 240   void reset() {
 241     _index = 0;
 242   }
 243 
 244   // Expand the stack, typically in response to an overflow condition.
 245   void expand();
 246 
 247   // Compute the least valued stack element.
 248   oop least_value(HeapWord* low) {
 249     HeapWord* least = low;
 250     for (size_t i = 0; i < _index; i++) {
 251       least = MIN2(least, (HeapWord*)_base[i]);
 252     }
 253     return (oop)least;
 254   }
 255 
 256   // Exposed here to allow stack expansion in || case.
 257   Mutex* par_lock() { return &_par_lock; }
 258 };
 259 
 260 class CardTableRS;
 261 class CMSParGCThreadState;
 262 
 263 class ModUnionClosure: public MemRegionClosure {
 264  protected:
 265   CMSBitMap* _t;
 266  public:
 267   ModUnionClosure(CMSBitMap* t): _t(t) { }
 268   void do_MemRegion(MemRegion mr);
 269 };
 270 
 271 class ModUnionClosurePar: public ModUnionClosure {
 272  public:
 273   ModUnionClosurePar(CMSBitMap* t): ModUnionClosure(t) { }
 274   void do_MemRegion(MemRegion mr);
 275 };
 276 
 277 // Survivor Chunk Array in support of parallelization of
 278 // Survivor Space rescan.
 279 class ChunkArray: public CHeapObj<mtGC> {
 280   size_t _index;
 281   size_t _capacity;
 282   size_t _overflows;
 283   HeapWord** _array;   // storage for array
 284 
 285  public:
 286   ChunkArray() : _index(0), _capacity(0), _overflows(0), _array(NULL) {}
 287   ChunkArray(HeapWord** a, size_t c):
 288     _index(0), _capacity(c), _overflows(0), _array(a) {}
 289 
 290   HeapWord** array() { return _array; }
 291   void set_array(HeapWord** a) { _array = a; }
 292 
 293   size_t capacity() { return _capacity; }
 294   void set_capacity(size_t c) { _capacity = c; }
 295 
 296   size_t end() {
 297     assert(_index <= capacity(),
 298            "_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT "): out of bounds",
 299            _index, _capacity);
 300     return _index;
 301   }  // exclusive
 302 
 303   HeapWord* nth(size_t n) {
 304     assert(n < end(), "Out of bounds access");
 305     return _array[n];
 306   }
 307 
 308   void reset() {
 309     _index = 0;
 310     if (_overflows > 0) {
 311       log_trace(gc)("CMS: ChunkArray[" SIZE_FORMAT "] overflowed " SIZE_FORMAT " times", _capacity, _overflows);
 312     }
 313     _overflows = 0;
 314   }
 315 
 316   void record_sample(HeapWord* p, size_t sz) {
 317     // For now we do not do anything with the size
 318     if (_index < _capacity) {
 319       _array[_index++] = p;
 320     } else {
 321       ++_overflows;
 322       assert(_index == _capacity,
 323              "_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT
 324              "): out of bounds at overflow#" SIZE_FORMAT,
 325              _index, _capacity, _overflows);
 326     }
 327   }
 328 };
 329 
 330 //
 331 // Timing, allocation and promotion statistics for gc scheduling and incremental
 332 // mode pacing.  Most statistics are exponential averages.
 333 //
 334 class CMSStats VALUE_OBJ_CLASS_SPEC {
 335  private:
 336   ConcurrentMarkSweepGeneration* const _cms_gen;   // The cms (old) gen.
 337 
 338   // The following are exponential averages with factor alpha:
 339   //   avg = (100 - alpha) * avg + alpha * cur_sample
 340   //
 341   //   The durations measure:  end_time[n] - start_time[n]
 342   //   The periods measure:    start_time[n] - start_time[n-1]
 343   //
 344   // The cms period and duration include only concurrent collections; time spent
 345   // in foreground cms collections due to System.gc() or because of a failure to
 346   // keep up are not included.
 347   //
 348   // There are 3 alphas to "bootstrap" the statistics.  The _saved_alpha is the
 349   // real value, but is used only after the first period.  A value of 100 is
 350   // used for the first sample so it gets the entire weight.
 351   unsigned int _saved_alpha; // 0-100
 352   unsigned int _gc0_alpha;
 353   unsigned int _cms_alpha;
 354 
 355   double _gc0_duration;
 356   double _gc0_period;
 357   size_t _gc0_promoted;         // bytes promoted per gc0
 358   double _cms_duration;
 359   double _cms_duration_pre_sweep; // time from initiation to start of sweep
 360   double _cms_period;
 361   size_t _cms_allocated;        // bytes of direct allocation per gc0 period
 362 
 363   // Timers.
 364   elapsedTimer _cms_timer;
 365   TimeStamp    _gc0_begin_time;
 366   TimeStamp    _cms_begin_time;
 367   TimeStamp    _cms_end_time;
 368 
 369   // Snapshots of the amount used in the CMS generation.
 370   size_t _cms_used_at_gc0_begin;
 371   size_t _cms_used_at_gc0_end;
 372   size_t _cms_used_at_cms_begin;
 373 
 374   // Used to prevent the duty cycle from being reduced in the middle of a cms
 375   // cycle.
 376   bool _allow_duty_cycle_reduction;
 377 
 378   enum {
 379     _GC0_VALID = 0x1,
 380     _CMS_VALID = 0x2,
 381     _ALL_VALID = _GC0_VALID | _CMS_VALID
 382   };
 383 
 384   unsigned int _valid_bits;
 385 
 386  protected:
 387   // In support of adjusting of cms trigger ratios based on history
 388   // of concurrent mode failure.
 389   double cms_free_adjustment_factor(size_t free) const;
 390   void   adjust_cms_free_adjustment_factor(bool fail, size_t free);
 391 
 392  public:
 393   CMSStats(ConcurrentMarkSweepGeneration* cms_gen,
 394            unsigned int alpha = CMSExpAvgFactor);
 395 
 396   // Whether or not the statistics contain valid data; higher level statistics
 397   // cannot be called until this returns true (they require at least one young
 398   // gen and one cms cycle to have completed).
 399   bool valid() const;
 400 
 401   // Record statistics.
 402   void record_gc0_begin();
 403   void record_gc0_end(size_t cms_gen_bytes_used);
 404   void record_cms_begin();
 405   void record_cms_end();
 406 
 407   // Allow management of the cms timer, which must be stopped/started around
 408   // yield points.
 409   elapsedTimer& cms_timer()     { return _cms_timer; }
 410   void start_cms_timer()        { _cms_timer.start(); }
 411   void stop_cms_timer()         { _cms_timer.stop(); }
 412 
 413   // Basic statistics; units are seconds or bytes.
 414   double gc0_period() const     { return _gc0_period; }
 415   double gc0_duration() const   { return _gc0_duration; }
 416   size_t gc0_promoted() const   { return _gc0_promoted; }
 417   double cms_period() const          { return _cms_period; }
 418   double cms_duration() const        { return _cms_duration; }
 419   size_t cms_allocated() const       { return _cms_allocated; }
 420 
 421   size_t cms_used_at_gc0_end() const { return _cms_used_at_gc0_end;}
 422 
 423   // Seconds since the last background cms cycle began or ended.
 424   double cms_time_since_begin() const;
 425   double cms_time_since_end() const;
 426 
 427   // Higher level statistics--caller must check that valid() returns true before
 428   // calling.
 429 
 430   // Returns bytes promoted per second of wall clock time.
 431   double promotion_rate() const;
 432 
 433   // Returns bytes directly allocated per second of wall clock time.
 434   double cms_allocation_rate() const;
 435 
 436   // Rate at which space in the cms generation is being consumed (sum of the
 437   // above two).
 438   double cms_consumption_rate() const;
 439 
 440   // Returns an estimate of the number of seconds until the cms generation will
 441   // fill up, assuming no collection work is done.
 442   double time_until_cms_gen_full() const;
 443 
 444   // Returns an estimate of the number of seconds remaining until
 445   // the cms generation collection should start.
 446   double time_until_cms_start() const;
 447 
 448   // End of higher level statistics.
 449 
 450   // Debugging.
 451   void print_on(outputStream* st) const PRODUCT_RETURN;
 452   void print() const { print_on(tty); }
 453 };
 454 
 455 // A closure related to weak references processing which
 456 // we embed in the CMSCollector, since we need to pass
 457 // it to the reference processor for secondary filtering
 458 // of references based on reachability of referent;
 459 // see role of _is_alive_non_header closure in the
 460 // ReferenceProcessor class.
 461 // For objects in the CMS generation, this closure checks
 462 // if the object is "live" (reachable). Used in weak
 463 // reference processing.
 464 class CMSIsAliveClosure: public BoolObjectClosure {
 465   const MemRegion  _span;
 466   const CMSBitMap* _bit_map;
 467 
 468   friend class CMSCollector;
 469  public:
 470   CMSIsAliveClosure(MemRegion span,
 471                     CMSBitMap* bit_map):
 472     _span(span),
 473     _bit_map(bit_map) {
 474     assert(!span.is_empty(), "Empty span could spell trouble");
 475   }
 476 
 477   bool do_object_b(oop obj);
 478 };
 479 
 480 
 481 // Implements AbstractRefProcTaskExecutor for CMS.
 482 class CMSRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
 483 public:
 484 
 485   CMSRefProcTaskExecutor(CMSCollector& collector)
 486     : _collector(collector)
 487   { }
 488 
 489   // Executes a task using worker threads.
 490   virtual void execute(ProcessTask& task);
 491   virtual void execute(EnqueueTask& task);
 492 private:
 493   CMSCollector& _collector;
 494 };
 495 
 496 
 497 class CMSCollector: public CHeapObj<mtGC> {
 498   friend class VMStructs;
 499   friend class ConcurrentMarkSweepThread;
 500   friend class ConcurrentMarkSweepGeneration;
 501   friend class CompactibleFreeListSpace;
 502   friend class CMSParMarkTask;
 503   friend class CMSParInitialMarkTask;
 504   friend class CMSParRemarkTask;
 505   friend class CMSConcMarkingTask;
 506   friend class CMSRefProcTaskProxy;
 507   friend class CMSRefProcTaskExecutor;
 508   friend class ScanMarkedObjectsAgainCarefullyClosure;  // for sampling eden
 509   friend class SurvivorSpacePrecleanClosure;            // --- ditto -------
 510   friend class PushOrMarkClosure;             // to access _restart_addr
 511   friend class ParPushOrMarkClosure;          // to access _restart_addr
 512   friend class MarkFromRootsClosure;          //  -- ditto --
 513                                               // ... and for clearing cards
 514   friend class ParMarkFromRootsClosure;       //  to access _restart_addr
 515                                               // ... and for clearing cards
 516   friend class ParConcMarkingClosure;         //  to access _restart_addr etc.
 517   friend class MarkFromRootsVerifyClosure;    // to access _restart_addr
 518   friend class PushAndMarkVerifyClosure;      //  -- ditto --
 519   friend class MarkRefsIntoAndScanClosure;    // to access _overflow_list
 520   friend class PushAndMarkClosure;            //  -- ditto --
 521   friend class ParPushAndMarkClosure;         //  -- ditto --
 522   friend class CMSKeepAliveClosure;           //  -- ditto --
 523   friend class CMSDrainMarkingStackClosure;   //  -- ditto --
 524   friend class CMSInnerParMarkAndPushClosure; //  -- ditto --
 525   NOT_PRODUCT(friend class ScanMarkedObjectsAgainClosure;) //  assertion on _overflow_list
 526   friend class ReleaseForegroundGC;  // to access _foregroundGCShouldWait
 527   friend class VM_CMS_Operation;
 528   friend class VM_CMS_Initial_Mark;
 529   friend class VM_CMS_Final_Remark;
 530   friend class TraceCMSMemoryManagerStats;
 531 
 532  private:
 533   jlong _time_of_last_gc;
 534   void update_time_of_last_gc(jlong now) {
 535     _time_of_last_gc = now;
 536   }
 537 
 538   OopTaskQueueSet* _task_queues;
 539 
 540   // Overflow list of grey objects, threaded through mark-word
 541   // Manipulated with CAS in the parallel/multi-threaded case.
 542   oopDesc* volatile _overflow_list;
 543   // The following array-pair keeps track of mark words
 544   // displaced for accommodating overflow list above.
 545   // This code will likely be revisited under RFE#4922830.
 546   Stack<oop, mtGC>     _preserved_oop_stack;
 547   Stack<markOop, mtGC> _preserved_mark_stack;
 548 
 549   int*             _hash_seed;
 550 
 551   // In support of multi-threaded concurrent phases
 552   YieldingFlexibleWorkGang* _conc_workers;
 553 
 554   // Performance Counters
 555   CollectorCounters* _gc_counters;
 556   CollectorCounters* _cgc_counters;
 557 
 558   // Initialization Errors
 559   bool _completed_initialization;
 560 
 561   // In support of ExplicitGCInvokesConcurrent
 562   static bool _full_gc_requested;
 563   static GCCause::Cause _full_gc_cause;
 564   unsigned int _collection_count_start;
 565 
 566   // Should we unload classes this concurrent cycle?
 567   bool _should_unload_classes;
 568   unsigned int  _concurrent_cycles_since_last_unload;
 569   unsigned int concurrent_cycles_since_last_unload() const {
 570     return _concurrent_cycles_since_last_unload;
 571   }
 572   // Did we (allow) unload classes in the previous concurrent cycle?
 573   bool unloaded_classes_last_cycle() const {
 574     return concurrent_cycles_since_last_unload() == 0;
 575   }
 576   // Root scanning options for perm gen
 577   int _roots_scanning_options;
 578   int roots_scanning_options() const      { return _roots_scanning_options; }
 579   void add_root_scanning_option(int o)    { _roots_scanning_options |= o;   }
 580   void remove_root_scanning_option(int o) { _roots_scanning_options &= ~o;  }
 581 
 582   // Verification support
 583   CMSBitMap     _verification_mark_bm;
 584   void verify_after_remark_work_1();
 585   void verify_after_remark_work_2();
 586 
 587   // True if any verification flag is on.
 588   bool _verifying;
 589   bool verifying() const { return _verifying; }
 590   void set_verifying(bool v) { _verifying = v; }
 591 
 592   // Collector policy
 593   ConcurrentMarkSweepPolicy* _collector_policy;
 594   ConcurrentMarkSweepPolicy* collector_policy() { return _collector_policy; }
 595 
 596   void set_did_compact(bool v);
 597 
 598   // XXX Move these to CMSStats ??? FIX ME !!!
 599   elapsedTimer _inter_sweep_timer;   // Time between sweeps
 600   elapsedTimer _intra_sweep_timer;   // Time _in_ sweeps
 601   // Padded decaying average estimates of the above
 602   AdaptivePaddedAverage _inter_sweep_estimate;
 603   AdaptivePaddedAverage _intra_sweep_estimate;
 604 
 605   CMSTracer* _gc_tracer_cm;
 606   ConcurrentGCTimer* _gc_timer_cm;
 607 
 608   bool _cms_start_registered;
 609 
 610   GCHeapSummary _last_heap_summary;
 611   MetaspaceSummary _last_metaspace_summary;
 612 
 613   void register_gc_start(GCCause::Cause cause);
 614   void register_gc_end();
 615   void save_heap_summary();
 616   void report_heap_summary(GCWhen::Type when);
 617 
 618  protected:
 619   ConcurrentMarkSweepGeneration* _cmsGen;  // Old gen (CMS)
 620   MemRegion                      _span;    // Span covering above two
 621   CardTableRS*                   _ct;      // Card table
 622 
 623   // CMS marking support structures
 624   CMSBitMap     _markBitMap;
 625   CMSBitMap     _modUnionTable;
 626   CMSMarkStack  _markStack;
 627 
 628   HeapWord*     _restart_addr; // In support of marking stack overflow
 629   void          lower_restart_addr(HeapWord* low);
 630 
 631   // Counters in support of marking stack / work queue overflow handling:
 632   // a non-zero value indicates certain types of overflow events during
 633   // the current CMS cycle and could lead to stack resizing efforts at
 634   // an opportune future time.
 635   size_t        _ser_pmc_preclean_ovflw;
 636   size_t        _ser_pmc_remark_ovflw;
 637   size_t        _par_pmc_remark_ovflw;
 638   size_t        _ser_kac_preclean_ovflw;
 639   size_t        _ser_kac_ovflw;
 640   size_t        _par_kac_ovflw;
 641   NOT_PRODUCT(ssize_t _num_par_pushes;)
 642 
 643   // ("Weak") Reference processing support.
 644   ReferenceProcessor*            _ref_processor;
 645   CMSIsAliveClosure              _is_alive_closure;
 646   // Keep this textually after _markBitMap and _span; c'tor dependency.
 647 
 648   ConcurrentMarkSweepThread*     _cmsThread;   // The thread doing the work
 649   ModUnionClosurePar _modUnionClosurePar;
 650 
 651   // CMS abstract state machine
 652   // initial_state: Idling
 653   // next_state(Idling)            = {Marking}
 654   // next_state(Marking)           = {Precleaning, Sweeping}
 655   // next_state(Precleaning)       = {AbortablePreclean, FinalMarking}
 656   // next_state(AbortablePreclean) = {FinalMarking}
 657   // next_state(FinalMarking)      = {Sweeping}
 658   // next_state(Sweeping)          = {Resizing}
 659   // next_state(Resizing)          = {Resetting}
 660   // next_state(Resetting)         = {Idling}
 661   // The numeric values below are chosen so that:
 662   // . _collectorState <= Idling ==  post-sweep && pre-mark
 663   // . _collectorState in (Idling, Sweeping) == {initial,final}marking ||
 664   //                                            precleaning || abortablePrecleanb
 665  public:
 666   enum CollectorState {
 667     Resizing            = 0,
 668     Resetting           = 1,
 669     Idling              = 2,
 670     InitialMarking      = 3,
 671     Marking             = 4,
 672     Precleaning         = 5,
 673     AbortablePreclean   = 6,
 674     FinalMarking        = 7,
 675     Sweeping            = 8
 676   };
 677  protected:
 678   static CollectorState _collectorState;
 679 
 680   // State related to prologue/epilogue invocation for my generations
 681   bool _between_prologue_and_epilogue;
 682 
 683   // Signaling/State related to coordination between fore- and background GC
 684   // Note: When the baton has been passed from background GC to foreground GC,
 685   // _foregroundGCIsActive is true and _foregroundGCShouldWait is false.
 686   static bool _foregroundGCIsActive;    // true iff foreground collector is active or
 687                                  // wants to go active
 688   static bool _foregroundGCShouldWait;  // true iff background GC is active and has not
 689                                  // yet passed the baton to the foreground GC
 690 
 691   // Support for CMSScheduleRemark (abortable preclean)
 692   bool _abort_preclean;
 693   bool _start_sampling;
 694 
 695   int    _numYields;
 696   size_t _numDirtyCards;
 697   size_t _sweep_count;
 698 
 699   // Occupancy used for bootstrapping stats
 700   double _bootstrap_occupancy;
 701 
 702   // Timer
 703   elapsedTimer _timer;
 704 
 705   // Timing, allocation and promotion statistics, used for scheduling.
 706   CMSStats      _stats;
 707 
 708   enum CMS_op_type {
 709     CMS_op_checkpointRootsInitial,
 710     CMS_op_checkpointRootsFinal
 711   };
 712 
 713   void do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause);
 714   bool stop_world_and_do(CMS_op_type op);
 715 
 716   OopTaskQueueSet* task_queues() { return _task_queues; }
 717   int*             hash_seed(int i) { return &_hash_seed[i]; }
 718   YieldingFlexibleWorkGang* conc_workers() { return _conc_workers; }
 719 
 720   // Support for parallelizing Eden rescan in CMS remark phase
 721   void sample_eden(); // ... sample Eden space top
 722 
 723  private:
 724   // Support for parallelizing young gen rescan in CMS remark phase
 725   ParNewGeneration* _young_gen;
 726 
 727   HeapWord* volatile* _top_addr;    // ... Top of Eden
 728   HeapWord**          _end_addr;    // ... End of Eden
 729   Mutex*              _eden_chunk_lock;
 730   HeapWord**          _eden_chunk_array; // ... Eden partitioning array
 731   size_t              _eden_chunk_index; // ... top (exclusive) of array
 732   size_t              _eden_chunk_capacity;  // ... max entries in array
 733 
 734   // Support for parallelizing survivor space rescan
 735   HeapWord** _survivor_chunk_array;
 736   size_t     _survivor_chunk_index;
 737   size_t     _survivor_chunk_capacity;
 738   size_t*    _cursor;
 739   ChunkArray* _survivor_plab_array;
 740 
 741   // Support for marking stack overflow handling
 742   bool take_from_overflow_list(size_t num, CMSMarkStack* to_stack);
 743   bool par_take_from_overflow_list(size_t num,
 744                                    OopTaskQueue* to_work_q,
 745                                    int no_of_gc_threads);
 746   void push_on_overflow_list(oop p);
 747   void par_push_on_overflow_list(oop p);
 748   // The following is, obviously, not, in general, "MT-stable"
 749   bool overflow_list_is_empty() const;
 750 
 751   void preserve_mark_if_necessary(oop p);
 752   void par_preserve_mark_if_necessary(oop p);
 753   void preserve_mark_work(oop p, markOop m);
 754   void restore_preserved_marks_if_any();
 755   NOT_PRODUCT(bool no_preserved_marks() const;)
 756   // In support of testing overflow code
 757   NOT_PRODUCT(int _overflow_counter;)
 758   NOT_PRODUCT(bool simulate_overflow();)       // Sequential
 759   NOT_PRODUCT(bool par_simulate_overflow();)   // MT version
 760 
 761   // CMS work methods
 762   void checkpointRootsInitialWork(); // Initial checkpoint work
 763 
 764   // A return value of false indicates failure due to stack overflow
 765   bool markFromRootsWork();  // Concurrent marking work
 766 
 767  public:   // FIX ME!!! only for testing
 768   bool do_marking_st();      // Single-threaded marking
 769   bool do_marking_mt();      // Multi-threaded  marking
 770 
 771  private:
 772 
 773   // Concurrent precleaning work
 774   size_t preclean_mod_union_table(ConcurrentMarkSweepGeneration* old_gen,
 775                                   ScanMarkedObjectsAgainCarefullyClosure* cl);
 776   size_t preclean_card_table(ConcurrentMarkSweepGeneration* old_gen,
 777                              ScanMarkedObjectsAgainCarefullyClosure* cl);
 778   // Does precleaning work, returning a quantity indicative of
 779   // the amount of "useful work" done.
 780   size_t preclean_work(bool clean_refs, bool clean_survivors);
 781   void preclean_cld(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock);
 782   void abortable_preclean(); // Preclean while looking for possible abort
 783   void initialize_sequential_subtasks_for_young_gen_rescan(int i);
 784   // Helper function for above; merge-sorts the per-thread plab samples
 785   void merge_survivor_plab_arrays(ContiguousSpace* surv, int no_of_gc_threads);
 786   // Resets (i.e. clears) the per-thread plab sample vectors
 787   void reset_survivor_plab_arrays();
 788 
 789   // Final (second) checkpoint work
 790   void checkpointRootsFinalWork();
 791   // Work routine for parallel version of remark
 792   void do_remark_parallel();
 793   // Work routine for non-parallel version of remark
 794   void do_remark_non_parallel();
 795   // Reference processing work routine (during second checkpoint)
 796   void refProcessingWork();
 797 
 798   // Concurrent sweeping work
 799   void sweepWork(ConcurrentMarkSweepGeneration* old_gen);
 800 
 801   // Concurrent resetting of support data structures
 802   void reset_concurrent();
 803   // Resetting of support data structures from a STW full GC
 804   void reset_stw();
 805 
 806   // Clear _expansion_cause fields of constituent generations
 807   void clear_expansion_cause();
 808 
 809   // An auxiliary method used to record the ends of
 810   // used regions of each generation to limit the extent of sweep
 811   void save_sweep_limits();
 812 
 813   // A work method used by the foreground collector to do
 814   // a mark-sweep-compact.
 815   void do_compaction_work(bool clear_all_soft_refs);
 816 
 817   // Work methods for reporting concurrent mode interruption or failure
 818   bool is_external_interruption();
 819   void report_concurrent_mode_interruption();
 820 
 821   // If the background GC is active, acquire control from the background
 822   // GC and do the collection.
 823   void acquire_control_and_collect(bool   full, bool clear_all_soft_refs);
 824 
 825   // For synchronizing passing of control from background to foreground
 826   // GC.  waitForForegroundGC() is called by the background
 827   // collector.  It if had to wait for a foreground collection,
 828   // it returns true and the background collection should assume
 829   // that the collection was finished by the foreground
 830   // collector.
 831   bool waitForForegroundGC();
 832 
 833   size_t block_size_using_printezis_bits(HeapWord* addr) const;
 834   size_t block_size_if_printezis_bits(HeapWord* addr) const;
 835   HeapWord* next_card_start_after_block(HeapWord* addr) const;
 836 
 837   void setup_cms_unloading_and_verification_state();
 838  public:
 839   CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
 840                CardTableRS*                   ct,
 841                ConcurrentMarkSweepPolicy*     cp);
 842   ConcurrentMarkSweepThread* cmsThread() { return _cmsThread; }
 843 
 844   ReferenceProcessor* ref_processor() { return _ref_processor; }
 845   void ref_processor_init();
 846 
 847   Mutex* bitMapLock()        const { return _markBitMap.lock();    }
 848   static CollectorState abstract_state() { return _collectorState;  }
 849 
 850   bool should_abort_preclean() const; // Whether preclean should be aborted.
 851   size_t get_eden_used() const;
 852   size_t get_eden_capacity() const;
 853 
 854   ConcurrentMarkSweepGeneration* cmsGen() { return _cmsGen; }
 855 
 856   // Locking checks
 857   NOT_PRODUCT(static bool have_cms_token();)
 858 
 859   bool shouldConcurrentCollect();
 860 
 861   void collect(bool   full,
 862                bool   clear_all_soft_refs,
 863                size_t size,
 864                bool   tlab);
 865   void collect_in_background(GCCause::Cause cause);
 866 
 867   // In support of ExplicitGCInvokesConcurrent
 868   static void request_full_gc(unsigned int full_gc_count, GCCause::Cause cause);
 869   // Should we unload classes in a particular concurrent cycle?
 870   bool should_unload_classes() const {
 871     return _should_unload_classes;
 872   }
 873   void update_should_unload_classes();
 874 
 875   void direct_allocated(HeapWord* start, size_t size);
 876 
 877   // Object is dead if not marked and current phase is sweeping.
 878   bool is_dead_obj(oop obj) const;
 879 
 880   // After a promotion (of "start"), do any necessary marking.
 881   // If "par", then it's being done by a parallel GC thread.
 882   // The last two args indicate if we need precise marking
 883   // and if so the size of the object so it can be dirtied
 884   // in its entirety.
 885   void promoted(bool par, HeapWord* start,
 886                 bool is_obj_array, size_t obj_size);
 887 
 888   void getFreelistLocks() const;
 889   void releaseFreelistLocks() const;
 890   bool haveFreelistLocks() const;
 891 
 892   // Adjust size of underlying generation
 893   void compute_new_size();
 894 
 895   // GC prologue and epilogue
 896   void gc_prologue(bool full);
 897   void gc_epilogue(bool full);
 898 
 899   jlong time_of_last_gc(jlong now) {
 900     if (_collectorState <= Idling) {
 901       // gc not in progress
 902       return _time_of_last_gc;
 903     } else {
 904       // collection in progress
 905       return now;
 906     }
 907   }
 908 
 909   // Support for parallel remark of survivor space
 910   void* get_data_recorder(int thr_num);
 911   void sample_eden_chunk();
 912 
 913   CMSBitMap* markBitMap()  { return &_markBitMap; }
 914   void directAllocated(HeapWord* start, size_t size);
 915 
 916   // Main CMS steps and related support
 917   void checkpointRootsInitial();
 918   bool markFromRoots();  // a return value of false indicates failure
 919                          // due to stack overflow
 920   void preclean();
 921   void checkpointRootsFinal();
 922   void sweep();
 923 
 924   // Check that the currently executing thread is the expected
 925   // one (foreground collector or background collector).
 926   static void check_correct_thread_executing() PRODUCT_RETURN;
 927 
 928   NOT_PRODUCT(bool is_cms_reachable(HeapWord* addr);)
 929 
 930   // Performance Counter Support
 931   CollectorCounters* counters()     { return _gc_counters; }
 932   CollectorCounters* cgc_counters() { return _cgc_counters; }
 933 
 934   // Timer stuff
 935   void    startTimer() { assert(!_timer.is_active(), "Error"); _timer.start();   }
 936   void    stopTimer()  { assert( _timer.is_active(), "Error"); _timer.stop();    }
 937   void    resetTimer() { assert(!_timer.is_active(), "Error"); _timer.reset();   }
 938   jlong   timerTicks() { assert(!_timer.is_active(), "Error"); return _timer.ticks(); }
 939 
 940   int  yields()          { return _numYields; }
 941   void resetYields()     { _numYields = 0;    }
 942   void incrementYields() { _numYields++;      }
 943   void resetNumDirtyCards()               { _numDirtyCards = 0; }
 944   void incrementNumDirtyCards(size_t num) { _numDirtyCards += num; }
 945   size_t  numDirtyCards()                 { return _numDirtyCards; }
 946 
 947   static bool foregroundGCShouldWait() { return _foregroundGCShouldWait; }
 948   static void set_foregroundGCShouldWait(bool v) { _foregroundGCShouldWait = v; }
 949   static bool foregroundGCIsActive() { return _foregroundGCIsActive; }
 950   static void set_foregroundGCIsActive(bool v) { _foregroundGCIsActive = v; }
 951   size_t sweep_count() const             { return _sweep_count; }
 952   void   increment_sweep_count()         { _sweep_count++; }
 953 
 954   // Timers/stats for gc scheduling and incremental mode pacing.
 955   CMSStats& stats() { return _stats; }
 956 
 957   // Adaptive size policy
 958   AdaptiveSizePolicy* size_policy();
 959 
 960   static void print_on_error(outputStream* st);
 961 
 962   // Debugging
 963   void verify();
 964   bool verify_after_remark();
 965   void verify_ok_to_terminate() const PRODUCT_RETURN;
 966   void verify_work_stacks_empty() const PRODUCT_RETURN;
 967   void verify_overflow_empty() const PRODUCT_RETURN;
 968 
 969   // Convenience methods in support of debugging
 970   static const size_t skip_header_HeapWords() PRODUCT_RETURN0;
 971   HeapWord* block_start(const void* p) const PRODUCT_RETURN0;
 972 
 973   // Accessors
 974   CMSMarkStack* verification_mark_stack() { return &_markStack; }
 975   CMSBitMap*    verification_mark_bm()    { return &_verification_mark_bm; }
 976 
 977   // Initialization errors
 978   bool completed_initialization() { return _completed_initialization; }
 979 
 980   void print_eden_and_survivor_chunk_arrays();
 981 
 982   ConcurrentGCTimer* gc_timer_cm() const { return _gc_timer_cm; }
 983 };
 984 
 985 class CMSExpansionCause : public AllStatic  {
 986  public:
 987   enum Cause {
 988     _no_expansion,
 989     _satisfy_free_ratio,
 990     _satisfy_promotion,
 991     _satisfy_allocation,
 992     _allocate_par_lab,
 993     _allocate_par_spooling_space,
 994     _adaptive_size_policy
 995   };
 996   // Return a string describing the cause of the expansion.
 997   static const char* to_string(CMSExpansionCause::Cause cause);
 998 };
 999 
1000 class ConcurrentMarkSweepGeneration: public CardGeneration {
1001   friend class VMStructs;
1002   friend class ConcurrentMarkSweepThread;
1003   friend class ConcurrentMarkSweep;
1004   friend class CMSCollector;
1005  protected:
1006   static CMSCollector*       _collector; // the collector that collects us
1007   CompactibleFreeListSpace*  _cmsSpace;  // underlying space (only one for now)
1008 
1009   // Performance Counters
1010   GenerationCounters*      _gen_counters;
1011   GSpaceCounters*          _space_counters;
1012 
1013   // Words directly allocated, used by CMSStats.
1014   size_t _direct_allocated_words;
1015 
1016   // Non-product stat counters
1017   NOT_PRODUCT(
1018     size_t _numObjectsPromoted;
1019     size_t _numWordsPromoted;
1020     size_t _numObjectsAllocated;
1021     size_t _numWordsAllocated;
1022   )
1023 
1024   // Used for sizing decisions
1025   bool _incremental_collection_failed;
1026   bool incremental_collection_failed() {
1027     return _incremental_collection_failed;
1028   }
1029   void set_incremental_collection_failed() {
1030     _incremental_collection_failed = true;
1031   }
1032   void clear_incremental_collection_failed() {
1033     _incremental_collection_failed = false;
1034   }
1035 
1036   // accessors
1037   void set_expansion_cause(CMSExpansionCause::Cause v) { _expansion_cause = v;}
1038   CMSExpansionCause::Cause expansion_cause() const { return _expansion_cause; }
1039 
1040   // Accessing spaces
1041   CompactibleSpace* space() const { return (CompactibleSpace*)_cmsSpace; }
1042 
1043  private:
1044   // For parallel young-gen GC support.
1045   CMSParGCThreadState** _par_gc_thread_states;
1046 
1047   // Reason generation was expanded
1048   CMSExpansionCause::Cause _expansion_cause;
1049 
1050   // In support of MinChunkSize being larger than min object size
1051   const double _dilatation_factor;
1052 
1053   // True if a compacting collection was done.
1054   bool _did_compact;
1055   bool did_compact() { return _did_compact; }
1056 
1057   // Fraction of current occupancy at which to start a CMS collection which
1058   // will collect this generation (at least).
1059   double _initiating_occupancy;
1060 
1061  protected:
1062   // Shrink generation by specified size (returns false if unable to shrink)
1063   void shrink_free_list_by(size_t bytes);
1064 
1065   // Update statistics for GC
1066   virtual void update_gc_stats(Generation* current_generation, bool full);
1067 
1068   // Maximum available space in the generation (including uncommitted)
1069   // space.
1070   size_t max_available() const;
1071 
1072   // getter and initializer for _initiating_occupancy field.
1073   double initiating_occupancy() const { return _initiating_occupancy; }
1074   void   init_initiating_occupancy(intx io, uintx tr);
1075 
1076   void expand_for_gc_cause(size_t bytes, size_t expand_bytes, CMSExpansionCause::Cause cause);
1077 
1078   void assert_correct_size_change_locking();
1079 
1080  public:
1081   ConcurrentMarkSweepGeneration(ReservedSpace rs, size_t initial_byte_size, CardTableRS* ct);
1082 
1083   // Accessors
1084   CMSCollector* collector() const { return _collector; }
1085   static void set_collector(CMSCollector* collector) {
1086     assert(_collector == NULL, "already set");
1087     _collector = collector;
1088   }
1089   CompactibleFreeListSpace*  cmsSpace() const { return _cmsSpace;  }
1090 
1091   Mutex* freelistLock() const;
1092 
1093   virtual Generation::Name kind() { return Generation::ConcurrentMarkSweep; }
1094 
1095   void set_did_compact(bool v) { _did_compact = v; }
1096 
1097   bool refs_discovery_is_atomic() const { return false; }
1098   bool refs_discovery_is_mt()     const {
1099     // Note: CMS does MT-discovery during the parallel-remark
1100     // phases. Use ReferenceProcessorMTMutator to make refs
1101     // discovery MT-safe during such phases or other parallel
1102     // discovery phases in the future. This may all go away
1103     // if/when we decide that refs discovery is sufficiently
1104     // rare that the cost of the CAS's involved is in the
1105     // noise. That's a measurement that should be done, and
1106     // the code simplified if that turns out to be the case.
1107     return ConcGCThreads > 1;
1108   }
1109 
1110   // Override
1111   virtual void ref_processor_init();
1112 
1113   void clear_expansion_cause() { _expansion_cause = CMSExpansionCause::_no_expansion; }
1114 
1115   // Space enquiries
1116   double occupancy() const { return ((double)used())/((double)capacity()); }
1117   size_t contiguous_available() const;
1118   size_t unsafe_max_alloc_nogc() const;
1119 
1120   // over-rides
1121   MemRegion used_region_at_save_marks() const;
1122 
1123   // Adjust quantities in the generation affected by
1124   // the compaction.
1125   void reset_after_compaction();
1126 
1127   // Allocation support
1128   HeapWord* allocate(size_t size, bool tlab);
1129   HeapWord* have_lock_and_allocate(size_t size, bool tlab);
1130   oop       promote(oop obj, size_t obj_size);
1131   HeapWord* par_allocate(size_t size, bool tlab) {
1132     return allocate(size, tlab);
1133   }
1134 
1135 
1136   // Used by CMSStats to track direct allocation.  The value is sampled and
1137   // reset after each young gen collection.
1138   size_t direct_allocated_words() const { return _direct_allocated_words; }
1139   void reset_direct_allocated_words()   { _direct_allocated_words = 0; }
1140 
1141   // Overrides for parallel promotion.
1142   virtual oop par_promote(int thread_num,
1143                           oop obj, markOop m, size_t word_sz);
1144   virtual void par_promote_alloc_done(int thread_num);
1145   virtual void par_oop_since_save_marks_iterate_done(int thread_num);
1146 
1147   virtual bool promotion_attempt_is_safe(size_t promotion_in_bytes) const;
1148 
1149   // Inform this (old) generation that a promotion failure was
1150   // encountered during a collection of the young generation.
1151   virtual void promotion_failure_occurred();
1152 
1153   bool should_collect(bool full, size_t size, bool tlab);
1154   virtual bool should_concurrent_collect() const;
1155   virtual bool is_too_full() const;
1156   void collect(bool   full,
1157                bool   clear_all_soft_refs,
1158                size_t size,
1159                bool   tlab);
1160 
1161   HeapWord* expand_and_allocate(size_t word_size,
1162                                 bool tlab,
1163                                 bool parallel = false);
1164 
1165   // GC prologue and epilogue
1166   void gc_prologue(bool full);
1167   void gc_prologue_work(bool full, bool registerClosure,
1168                         ModUnionClosure* modUnionClosure);
1169   void gc_epilogue(bool full);
1170   void gc_epilogue_work(bool full);
1171 
1172   // Time since last GC of this generation
1173   jlong time_of_last_gc(jlong now) {
1174     return collector()->time_of_last_gc(now);
1175   }
1176   void update_time_of_last_gc(jlong now) {
1177     collector()-> update_time_of_last_gc(now);
1178   }
1179 
1180   // Allocation failure
1181   void shrink(size_t bytes);
1182   HeapWord* expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz);
1183   bool expand_and_ensure_spooling_space(PromotionInfo* promo);
1184 
1185   // Iteration support and related enquiries
1186   void save_marks();
1187   bool no_allocs_since_save_marks();
1188 
1189   // Iteration support specific to CMS generations
1190   void save_sweep_limit();
1191 
1192   // More iteration support
1193   virtual void oop_iterate(ExtendedOopClosure* cl);
1194   virtual void safe_object_iterate(ObjectClosure* cl);
1195   virtual void object_iterate(ObjectClosure* cl);
1196 
1197   // Need to declare the full complement of closures, whether we'll
1198   // override them or not, or get message from the compiler:
1199   //   oop_since_save_marks_iterate_nv hides virtual function...
1200   #define CMS_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \
1201     void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl);
1202   ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DECL)
1203 
1204   // Smart allocation  XXX -- move to CFLSpace?
1205   void setNearLargestChunk();
1206   bool isNearLargestChunk(HeapWord* addr);
1207 
1208   // Get the chunk at the end of the space.  Delegates to
1209   // the space.
1210   FreeChunk* find_chunk_at_end();
1211 
1212   void post_compact();
1213 
1214   // Debugging
1215   void prepare_for_verify();
1216   void verify();
1217   void print_statistics()               PRODUCT_RETURN;
1218 
1219   // Performance Counters support
1220   virtual void update_counters();
1221   virtual void update_counters(size_t used);
1222   void initialize_performance_counters();
1223   CollectorCounters* counters()  { return collector()->counters(); }
1224 
1225   // Support for parallel remark of survivor space
1226   void* get_data_recorder(int thr_num) {
1227     //Delegate to collector
1228     return collector()->get_data_recorder(thr_num);
1229   }
1230   void sample_eden_chunk() {
1231     //Delegate to collector
1232     return collector()->sample_eden_chunk();
1233   }
1234 
1235   // Printing
1236   const char* name() const;
1237   virtual const char* short_name() const { return "CMS"; }
1238   void        print() const;
1239 
1240   // Resize the generation after a compacting GC.  The
1241   // generation can be treated as a contiguous space
1242   // after the compaction.
1243   virtual void compute_new_size();
1244   // Resize the generation after a non-compacting
1245   // collection.
1246   void compute_new_size_free_list();
1247 };
1248 
1249 //
1250 // Closures of various sorts used by CMS to accomplish its work
1251 //
1252 
1253 // This closure is used to do concurrent marking from the roots
1254 // following the first checkpoint.
1255 class MarkFromRootsClosure: public BitMapClosure {
1256   CMSCollector*  _collector;
1257   MemRegion      _span;
1258   CMSBitMap*     _bitMap;
1259   CMSBitMap*     _mut;
1260   CMSMarkStack*  _markStack;
1261   bool           _yield;
1262   int            _skipBits;
1263   HeapWord*      _finger;
1264   HeapWord*      _threshold;
1265   DEBUG_ONLY(bool _verifying;)
1266 
1267  public:
1268   MarkFromRootsClosure(CMSCollector* collector, MemRegion span,
1269                        CMSBitMap* bitMap,
1270                        CMSMarkStack*  markStack,
1271                        bool should_yield, bool verifying = false);
1272   bool do_bit(size_t offset);
1273   void reset(HeapWord* addr);
1274   inline void do_yield_check();
1275 
1276  private:
1277   void scanOopsInOop(HeapWord* ptr);
1278   void do_yield_work();
1279 };
1280 
1281 // This closure is used to do concurrent multi-threaded
1282 // marking from the roots following the first checkpoint.
1283 // XXX This should really be a subclass of The serial version
1284 // above, but i have not had the time to refactor things cleanly.
1285 class ParMarkFromRootsClosure: public BitMapClosure {
1286   CMSCollector*  _collector;
1287   MemRegion      _whole_span;
1288   MemRegion      _span;
1289   CMSBitMap*     _bit_map;
1290   CMSBitMap*     _mut;
1291   OopTaskQueue*  _work_queue;
1292   CMSMarkStack*  _overflow_stack;
1293   int            _skip_bits;
1294   HeapWord*      _finger;
1295   HeapWord*      _threshold;
1296   CMSConcMarkingTask* _task;
1297  public:
1298   ParMarkFromRootsClosure(CMSConcMarkingTask* task, CMSCollector* collector,
1299                           MemRegion span,
1300                           CMSBitMap* bit_map,
1301                           OopTaskQueue* work_queue,
1302                           CMSMarkStack*  overflow_stack);
1303   bool do_bit(size_t offset);
1304   inline void do_yield_check();
1305 
1306  private:
1307   void scan_oops_in_oop(HeapWord* ptr);
1308   void do_yield_work();
1309   bool get_work_from_overflow_stack();
1310 };
1311 
1312 // The following closures are used to do certain kinds of verification of
1313 // CMS marking.
1314 class PushAndMarkVerifyClosure: public MetadataAwareOopClosure {
1315   CMSCollector*    _collector;
1316   MemRegion        _span;
1317   CMSBitMap*       _verification_bm;
1318   CMSBitMap*       _cms_bm;
1319   CMSMarkStack*    _mark_stack;
1320  protected:
1321   void do_oop(oop p);
1322   template <class T> inline void do_oop_work(T *p) {
1323     oop obj = oopDesc::load_decode_heap_oop(p);
1324     do_oop(obj);
1325   }
1326  public:
1327   PushAndMarkVerifyClosure(CMSCollector* cms_collector,
1328                            MemRegion span,
1329                            CMSBitMap* verification_bm,
1330                            CMSBitMap* cms_bm,
1331                            CMSMarkStack*  mark_stack);
1332   void do_oop(oop* p);
1333   void do_oop(narrowOop* p);
1334 
1335   // Deal with a stack overflow condition
1336   void handle_stack_overflow(HeapWord* lost);
1337 };
1338 
1339 class MarkFromRootsVerifyClosure: public BitMapClosure {
1340   CMSCollector*  _collector;
1341   MemRegion      _span;
1342   CMSBitMap*     _verification_bm;
1343   CMSBitMap*     _cms_bm;
1344   CMSMarkStack*  _mark_stack;
1345   HeapWord*      _finger;
1346   PushAndMarkVerifyClosure _pam_verify_closure;
1347  public:
1348   MarkFromRootsVerifyClosure(CMSCollector* collector, MemRegion span,
1349                              CMSBitMap* verification_bm,
1350                              CMSBitMap* cms_bm,
1351                              CMSMarkStack*  mark_stack);
1352   bool do_bit(size_t offset);
1353   void reset(HeapWord* addr);
1354 };
1355 
1356 
1357 // This closure is used to check that a certain set of bits is
1358 // "empty" (i.e. the bit vector doesn't have any 1-bits).
1359 class FalseBitMapClosure: public BitMapClosure {
1360  public:
1361   bool do_bit(size_t offset) {
1362     guarantee(false, "Should not have a 1 bit");
1363     return true;
1364   }
1365 };
1366 
1367 // A version of ObjectClosure with "memory" (see _previous_address below)
1368 class UpwardsObjectClosure: public BoolObjectClosure {
1369   HeapWord* _previous_address;
1370  public:
1371   UpwardsObjectClosure() : _previous_address(NULL) { }
1372   void set_previous(HeapWord* addr) { _previous_address = addr; }
1373   HeapWord* previous()              { return _previous_address; }
1374   // A return value of "true" can be used by the caller to decide
1375   // if this object's end should *NOT* be recorded in
1376   // _previous_address above.
1377   virtual bool do_object_bm(oop obj, MemRegion mr) = 0;
1378 };
1379 
1380 // This closure is used during the second checkpointing phase
1381 // to rescan the marked objects on the dirty cards in the mod
1382 // union table and the card table proper. It's invoked via
1383 // MarkFromDirtyCardsClosure below. It uses either
1384 // [Par_]MarkRefsIntoAndScanClosure (Par_ in the parallel case)
1385 // declared in genOopClosures.hpp to accomplish some of its work.
1386 // In the parallel case the bitMap is shared, so access to
1387 // it needs to be suitably synchronized for updates by embedded
1388 // closures that update it; however, this closure itself only
1389 // reads the bit_map and because it is idempotent, is immune to
1390 // reading stale values.
1391 class ScanMarkedObjectsAgainClosure: public UpwardsObjectClosure {
1392   #ifdef ASSERT
1393     CMSCollector*          _collector;
1394     MemRegion              _span;
1395     union {
1396       CMSMarkStack*        _mark_stack;
1397       OopTaskQueue*        _work_queue;
1398     };
1399   #endif // ASSERT
1400   bool                       _parallel;
1401   CMSBitMap*                 _bit_map;
1402   union {
1403     MarkRefsIntoAndScanClosure*    _scan_closure;
1404     ParMarkRefsIntoAndScanClosure* _par_scan_closure;
1405   };
1406 
1407  public:
1408   ScanMarkedObjectsAgainClosure(CMSCollector* collector,
1409                                 MemRegion span,
1410                                 ReferenceProcessor* rp,
1411                                 CMSBitMap* bit_map,
1412                                 CMSMarkStack*  mark_stack,
1413                                 MarkRefsIntoAndScanClosure* cl):
1414     #ifdef ASSERT
1415       _collector(collector),
1416       _span(span),
1417       _mark_stack(mark_stack),
1418     #endif // ASSERT
1419     _parallel(false),
1420     _bit_map(bit_map),
1421     _scan_closure(cl) { }
1422 
1423   ScanMarkedObjectsAgainClosure(CMSCollector* collector,
1424                                 MemRegion span,
1425                                 ReferenceProcessor* rp,
1426                                 CMSBitMap* bit_map,
1427                                 OopTaskQueue* work_queue,
1428                                 ParMarkRefsIntoAndScanClosure* cl):
1429     #ifdef ASSERT
1430       _collector(collector),
1431       _span(span),
1432       _work_queue(work_queue),
1433     #endif // ASSERT
1434     _parallel(true),
1435     _bit_map(bit_map),
1436     _par_scan_closure(cl) { }
1437 
1438   bool do_object_b(oop obj) {
1439     guarantee(false, "Call do_object_b(oop, MemRegion) form instead");
1440     return false;
1441   }
1442   bool do_object_bm(oop p, MemRegion mr);
1443 };
1444 
1445 // This closure is used during the second checkpointing phase
1446 // to rescan the marked objects on the dirty cards in the mod
1447 // union table and the card table proper. It invokes
1448 // ScanMarkedObjectsAgainClosure above to accomplish much of its work.
1449 // In the parallel case, the bit map is shared and requires
1450 // synchronized access.
1451 class MarkFromDirtyCardsClosure: public MemRegionClosure {
1452   CompactibleFreeListSpace*      _space;
1453   ScanMarkedObjectsAgainClosure  _scan_cl;
1454   size_t                         _num_dirty_cards;
1455 
1456  public:
1457   MarkFromDirtyCardsClosure(CMSCollector* collector,
1458                             MemRegion span,
1459                             CompactibleFreeListSpace* space,
1460                             CMSBitMap* bit_map,
1461                             CMSMarkStack* mark_stack,
1462                             MarkRefsIntoAndScanClosure* cl):
1463     _space(space),
1464     _num_dirty_cards(0),
1465     _scan_cl(collector, span, collector->ref_processor(), bit_map,
1466                  mark_stack, cl) { }
1467 
1468   MarkFromDirtyCardsClosure(CMSCollector* collector,
1469                             MemRegion span,
1470                             CompactibleFreeListSpace* space,
1471                             CMSBitMap* bit_map,
1472                             OopTaskQueue* work_queue,
1473                             ParMarkRefsIntoAndScanClosure* cl):
1474     _space(space),
1475     _num_dirty_cards(0),
1476     _scan_cl(collector, span, collector->ref_processor(), bit_map,
1477              work_queue, cl) { }
1478 
1479   void do_MemRegion(MemRegion mr);
1480   void set_space(CompactibleFreeListSpace* space) { _space = space; }
1481   size_t num_dirty_cards() { return _num_dirty_cards; }
1482 };
1483 
1484 // This closure is used in the non-product build to check
1485 // that there are no MemRegions with a certain property.
1486 class FalseMemRegionClosure: public MemRegionClosure {
1487   void do_MemRegion(MemRegion mr) {
1488     guarantee(!mr.is_empty(), "Shouldn't be empty");
1489     guarantee(false, "Should never be here");
1490   }
1491 };
1492 
1493 // This closure is used during the precleaning phase
1494 // to "carefully" rescan marked objects on dirty cards.
1495 // It uses MarkRefsIntoAndScanClosure declared in genOopClosures.hpp
1496 // to accomplish some of its work.
1497 class ScanMarkedObjectsAgainCarefullyClosure: public ObjectClosureCareful {
1498   CMSCollector*                  _collector;
1499   MemRegion                      _span;
1500   bool                           _yield;
1501   Mutex*                         _freelistLock;
1502   CMSBitMap*                     _bitMap;
1503   CMSMarkStack*                  _markStack;
1504   MarkRefsIntoAndScanClosure*    _scanningClosure;
1505   DEBUG_ONLY(HeapWord*           _last_scanned_object;)
1506 
1507  public:
1508   ScanMarkedObjectsAgainCarefullyClosure(CMSCollector* collector,
1509                                          MemRegion     span,
1510                                          CMSBitMap* bitMap,
1511                                          CMSMarkStack*  markStack,
1512                                          MarkRefsIntoAndScanClosure* cl,
1513                                          bool should_yield):
1514     _collector(collector),
1515     _span(span),
1516     _yield(should_yield),
1517     _bitMap(bitMap),
1518     _markStack(markStack),
1519     _scanningClosure(cl)
1520     DEBUG_ONLY(COMMA _last_scanned_object(NULL))
1521   { }
1522 
1523   void do_object(oop p) {
1524     guarantee(false, "call do_object_careful instead");
1525   }
1526 
1527   size_t      do_object_careful(oop p) {
1528     guarantee(false, "Unexpected caller");
1529     return 0;
1530   }
1531 
1532   size_t      do_object_careful_m(oop p, MemRegion mr);
1533 
1534   void setFreelistLock(Mutex* m) {
1535     _freelistLock = m;
1536     _scanningClosure->set_freelistLock(m);
1537   }
1538 
1539  private:
1540   inline bool do_yield_check();
1541 
1542   void do_yield_work();
1543 };
1544 
1545 class SurvivorSpacePrecleanClosure: public ObjectClosureCareful {
1546   CMSCollector*                  _collector;
1547   MemRegion                      _span;
1548   bool                           _yield;
1549   CMSBitMap*                     _bit_map;
1550   CMSMarkStack*                  _mark_stack;
1551   PushAndMarkClosure*            _scanning_closure;
1552   unsigned int                   _before_count;
1553 
1554  public:
1555   SurvivorSpacePrecleanClosure(CMSCollector* collector,
1556                                MemRegion     span,
1557                                CMSBitMap*    bit_map,
1558                                CMSMarkStack* mark_stack,
1559                                PushAndMarkClosure* cl,
1560                                unsigned int  before_count,
1561                                bool          should_yield):
1562     _collector(collector),
1563     _span(span),
1564     _yield(should_yield),
1565     _bit_map(bit_map),
1566     _mark_stack(mark_stack),
1567     _scanning_closure(cl),
1568     _before_count(before_count)
1569   { }
1570 
1571   void do_object(oop p) {
1572     guarantee(false, "call do_object_careful instead");
1573   }
1574 
1575   size_t      do_object_careful(oop p);
1576 
1577   size_t      do_object_careful_m(oop p, MemRegion mr) {
1578     guarantee(false, "Unexpected caller");
1579     return 0;
1580   }
1581 
1582  private:
1583   inline void do_yield_check();
1584   void do_yield_work();
1585 };
1586 
1587 // This closure is used to accomplish the sweeping work
1588 // after the second checkpoint but before the concurrent reset
1589 // phase.
1590 //
1591 // Terminology
1592 //   left hand chunk (LHC) - block of one or more chunks currently being
1593 //     coalesced.  The LHC is available for coalescing with a new chunk.
1594 //   right hand chunk (RHC) - block that is currently being swept that is
1595 //     free or garbage that can be coalesced with the LHC.
1596 // _inFreeRange is true if there is currently a LHC
1597 // _lastFreeRangeCoalesced is true if the LHC consists of more than one chunk.
1598 // _freeRangeInFreeLists is true if the LHC is in the free lists.
1599 // _freeFinger is the address of the current LHC
1600 class SweepClosure: public BlkClosureCareful {
1601   CMSCollector*                  _collector;  // collector doing the work
1602   ConcurrentMarkSweepGeneration* _g;    // Generation being swept
1603   CompactibleFreeListSpace*      _sp;   // Space being swept
1604   HeapWord*                      _limit;// the address at or above which the sweep should stop
1605                                         // because we do not expect newly garbage blocks
1606                                         // eligible for sweeping past that address.
1607   Mutex*                         _freelistLock; // Free list lock (in space)
1608   CMSBitMap*                     _bitMap;       // Marking bit map (in
1609                                                 // generation)
1610   bool                           _inFreeRange;  // Indicates if we are in the
1611                                                 // midst of a free run
1612   bool                           _freeRangeInFreeLists;
1613                                         // Often, we have just found
1614                                         // a free chunk and started
1615                                         // a new free range; we do not
1616                                         // eagerly remove this chunk from
1617                                         // the free lists unless there is
1618                                         // a possibility of coalescing.
1619                                         // When true, this flag indicates
1620                                         // that the _freeFinger below
1621                                         // points to a potentially free chunk
1622                                         // that may still be in the free lists
1623   bool                           _lastFreeRangeCoalesced;
1624                                         // free range contains chunks
1625                                         // coalesced
1626   bool                           _yield;
1627                                         // Whether sweeping should be
1628                                         // done with yields. For instance
1629                                         // when done by the foreground
1630                                         // collector we shouldn't yield.
1631   HeapWord*                      _freeFinger;   // When _inFreeRange is set, the
1632                                                 // pointer to the "left hand
1633                                                 // chunk"
1634   size_t                         _freeRangeSize;
1635                                         // When _inFreeRange is set, this
1636                                         // indicates the accumulated size
1637                                         // of the "left hand chunk"
1638   NOT_PRODUCT(
1639     size_t                       _numObjectsFreed;
1640     size_t                       _numWordsFreed;
1641     size_t                       _numObjectsLive;
1642     size_t                       _numWordsLive;
1643     size_t                       _numObjectsAlreadyFree;
1644     size_t                       _numWordsAlreadyFree;
1645     FreeChunk*                   _last_fc;
1646   )
1647  private:
1648   // Code that is common to a free chunk or garbage when
1649   // encountered during sweeping.
1650   void do_post_free_or_garbage_chunk(FreeChunk *fc, size_t chunkSize);
1651   // Process a free chunk during sweeping.
1652   void do_already_free_chunk(FreeChunk *fc);
1653   // Work method called when processing an already free or a
1654   // freshly garbage chunk to do a lookahead and possibly a
1655   // preemptive flush if crossing over _limit.
1656   void lookahead_and_flush(FreeChunk* fc, size_t chunkSize);
1657   // Process a garbage chunk during sweeping.
1658   size_t do_garbage_chunk(FreeChunk *fc);
1659   // Process a live chunk during sweeping.
1660   size_t do_live_chunk(FreeChunk* fc);
1661 
1662   // Accessors.
1663   HeapWord* freeFinger() const          { return _freeFinger; }
1664   void set_freeFinger(HeapWord* v)      { _freeFinger = v; }
1665   bool inFreeRange()    const           { return _inFreeRange; }
1666   void set_inFreeRange(bool v)          { _inFreeRange = v; }
1667   bool lastFreeRangeCoalesced() const    { return _lastFreeRangeCoalesced; }
1668   void set_lastFreeRangeCoalesced(bool v) { _lastFreeRangeCoalesced = v; }
1669   bool freeRangeInFreeLists() const     { return _freeRangeInFreeLists; }
1670   void set_freeRangeInFreeLists(bool v) { _freeRangeInFreeLists = v; }
1671 
1672   // Initialize a free range.
1673   void initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists);
1674   // Return this chunk to the free lists.
1675   void flush_cur_free_chunk(HeapWord* chunk, size_t size);
1676 
1677   // Check if we should yield and do so when necessary.
1678   inline void do_yield_check(HeapWord* addr);
1679 
1680   // Yield
1681   void do_yield_work(HeapWord* addr);
1682 
1683   // Debugging/Printing
1684   void print_free_block_coalesced(FreeChunk* fc) const;
1685 
1686  public:
1687   SweepClosure(CMSCollector* collector, ConcurrentMarkSweepGeneration* g,
1688                CMSBitMap* bitMap, bool should_yield);
1689   ~SweepClosure() PRODUCT_RETURN;
1690 
1691   size_t       do_blk_careful(HeapWord* addr);
1692   void         print() const { print_on(tty); }
1693   void         print_on(outputStream *st) const;
1694 };
1695 
1696 // Closures related to weak references processing
1697 
1698 // During CMS' weak reference processing, this is a
1699 // work-routine/closure used to complete transitive
1700 // marking of objects as live after a certain point
1701 // in which an initial set has been completely accumulated.
1702 // This closure is currently used both during the final
1703 // remark stop-world phase, as well as during the concurrent
1704 // precleaning of the discovered reference lists.
1705 class CMSDrainMarkingStackClosure: public VoidClosure {
1706   CMSCollector*        _collector;
1707   MemRegion            _span;
1708   CMSMarkStack*        _mark_stack;
1709   CMSBitMap*           _bit_map;
1710   CMSKeepAliveClosure* _keep_alive;
1711   bool                 _concurrent_precleaning;
1712  public:
1713   CMSDrainMarkingStackClosure(CMSCollector* collector, MemRegion span,
1714                       CMSBitMap* bit_map, CMSMarkStack* mark_stack,
1715                       CMSKeepAliveClosure* keep_alive,
1716                       bool cpc):
1717     _collector(collector),
1718     _span(span),
1719     _bit_map(bit_map),
1720     _mark_stack(mark_stack),
1721     _keep_alive(keep_alive),
1722     _concurrent_precleaning(cpc) {
1723     assert(_concurrent_precleaning == _keep_alive->concurrent_precleaning(),
1724            "Mismatch");
1725   }
1726 
1727   void do_void();
1728 };
1729 
1730 // A parallel version of CMSDrainMarkingStackClosure above.
1731 class CMSParDrainMarkingStackClosure: public VoidClosure {
1732   CMSCollector*           _collector;
1733   MemRegion               _span;
1734   OopTaskQueue*           _work_queue;
1735   CMSBitMap*              _bit_map;
1736   CMSInnerParMarkAndPushClosure _mark_and_push;
1737 
1738  public:
1739   CMSParDrainMarkingStackClosure(CMSCollector* collector,
1740                                  MemRegion span, CMSBitMap* bit_map,
1741                                  OopTaskQueue* work_queue):
1742     _collector(collector),
1743     _span(span),
1744     _bit_map(bit_map),
1745     _work_queue(work_queue),
1746     _mark_and_push(collector, span, bit_map, work_queue) { }
1747 
1748  public:
1749   void trim_queue(uint max);
1750   void do_void();
1751 };
1752 
1753 // Allow yielding or short-circuiting of reference list
1754 // precleaning work.
1755 class CMSPrecleanRefsYieldClosure: public YieldClosure {
1756   CMSCollector* _collector;
1757   void do_yield_work();
1758  public:
1759   CMSPrecleanRefsYieldClosure(CMSCollector* collector):
1760     _collector(collector) {}
1761   virtual bool should_return();
1762 };
1763 
1764 
1765 // Convenience class that locks free list locks for given CMS collector
1766 class FreelistLocker: public StackObj {
1767  private:
1768   CMSCollector* _collector;
1769  public:
1770   FreelistLocker(CMSCollector* collector):
1771     _collector(collector) {
1772     _collector->getFreelistLocks();
1773   }
1774 
1775   ~FreelistLocker() {
1776     _collector->releaseFreelistLocks();
1777   }
1778 };
1779 
1780 // Mark all dead objects in a given space.
1781 class MarkDeadObjectsClosure: public BlkClosure {
1782   const CMSCollector*             _collector;
1783   const CompactibleFreeListSpace* _sp;
1784   CMSBitMap*                      _live_bit_map;
1785   CMSBitMap*                      _dead_bit_map;
1786 public:
1787   MarkDeadObjectsClosure(const CMSCollector* collector,
1788                          const CompactibleFreeListSpace* sp,
1789                          CMSBitMap *live_bit_map,
1790                          CMSBitMap *dead_bit_map) :
1791     _collector(collector),
1792     _sp(sp),
1793     _live_bit_map(live_bit_map),
1794     _dead_bit_map(dead_bit_map) {}
1795   size_t do_blk(HeapWord* addr);
1796 };
1797 
1798 class TraceCMSMemoryManagerStats : public TraceMemoryManagerStats {
1799 
1800  public:
1801   TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause);
1802 };
1803 
1804 
1805 #endif // SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_HPP