--- old/src/hotspot/share/gc/cms/cmsHeap.cpp	Fri Jun 21 17:34:02 2019
+++ new/src/hotspot/share/gc/cms/cmsHeap.cpp	Fri Jun 21 17:34:01 2019
@@ -62,7 +62,7 @@
   }
 
   size_t used_in_bytes() {
-    return _space->used();
+    return _space->used_stable();
   }
 };
 
--- old/src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp	Fri Jun 21 17:34:04 2019
+++ new/src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp	Fri Jun 21 17:34:03 2019
@@ -372,6 +372,8 @@
     )
   }
   _dictionary->set_par_lock(&_parDictionaryAllocLock);
+
+  _used_stable = 0;
 }
 
 // Like CompactibleSpace forward() but always calls cross_threshold() to
@@ -577,6 +579,14 @@
   return capacity() - free();
 }
 
+size_t CompactibleFreeListSpace::used_stable() const {
+  return _used_stable;
+}
+
+void CompactibleFreeListSpace::recalculate_used_stable() {
+  _used_stable = used();
+}
+
 size_t CompactibleFreeListSpace::free() const {
   // "MT-safe, but not MT-precise"(TM), if you will: i.e.
   // if you do this while the structures are in flux you
@@ -1374,6 +1384,9 @@
     debug_only(fc->mangleAllocated(size));
   }
 
+  // After allocation, recalculate used space and update used_stable
+  recalculate_used_stable();
+
   return res;
 }
 
--- old/src/hotspot/share/gc/cms/compactibleFreeListSpace.hpp	Fri Jun 21 17:34:06 2019
+++ new/src/hotspot/share/gc/cms/compactibleFreeListSpace.hpp	Fri Jun 21 17:34:05 2019
@@ -192,6 +192,9 @@
   // Used to keep track of limit of sweep for the space
   HeapWord* _sweep_limit;
 
+  // Stable value of used().
+  size_t _used_stable;
+
   // Used to make the young collector update the mod union table
   MemRegionClosure* _preconsumptionDirtyCardClosure;
 
@@ -412,6 +415,17 @@
   // which overestimates the region by returning the entire
   // committed region (this is safe, but inefficient).
 
+  // Returns monotonically increasing stable used space bytes for CMS.
+  // This is required for jhat and other memory monitoring tools
+  // that might otherwise see inconsistent used space values during a garbage 
+  // collection, promotion or allocation into compactibleFreeListSpace.
+  // The value returned by this function might be smaller than the
+  // actual value.
+  size_t used_stable() const;
+  // Recalculate and cache the current stable used() value. Only to be called
+  // in places where we can be sure that the result is stable.
+  void recalculate_used_stable();
+
   // Returns a subregion of the space containing all the objects in
   // the space.
   MemRegion used_region() const {
--- old/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp	Fri Jun 21 17:34:08 2019
+++ new/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp	Fri Jun 21 17:34:07 2019
@@ -692,6 +692,10 @@
   return _cmsSpace->max_alloc_in_words() * HeapWordSize;
 }
 
+size_t ConcurrentMarkSweepGeneration::used_stable() const {
+  return cmsSpace()->used_stable();
+}
+
 size_t ConcurrentMarkSweepGeneration::max_available() const {
   return free() + _virtual_space.uncommitted_size();
 }
@@ -1523,6 +1527,8 @@
   FreelistLocker z(this);
   MetaspaceGC::compute_new_size();
   _cmsGen->compute_new_size_free_list();
+  // recalculate CMS used space after CMS collection
+  _cmsGen->cmsSpace()->recalculate_used_stable();
 }
 
 // A work method used by the foreground collector to do
@@ -1745,6 +1751,7 @@
   }
 
   size_t prev_used = _cmsGen->used();
+  _cmsGen->cmsSpace()->recalculate_used_stable();
 
   // The change of the collection state is normally done at this level;
   // the exceptions are phases that are executed while the world is
@@ -2051,6 +2058,7 @@
 
   _capacity_at_prologue = capacity();
   _used_at_prologue = used();
+  _cmsSpace->recalculate_used_stable();
 
   // We enable promotion tracking so that card-scanning can recognize
   // which objects have been promoted during this GC and skip them.
@@ -2123,6 +2131,7 @@
   _eden_chunk_index = 0;
 
   size_t cms_used   = _cmsGen->cmsSpace()->used();
+  _cmsGen->cmsSpace()->recalculate_used_stable();
 
   // update performance counters - this uses a special version of
   // update_counters() that allows the utilization to be passed as a
@@ -2816,6 +2825,8 @@
     rp->enable_discovery();
     _collectorState = Marking;
   }
+
+  _cmsGen->cmsSpace()->recalculate_used_stable();
 }
 
 void CMSCollector::checkpointRootsInitialWork() {
@@ -4177,6 +4188,7 @@
     MutexLocker y(bitMapLock(),
                   Mutex::_no_safepoint_check_flag);
     checkpointRootsFinalWork();
+    _cmsGen->cmsSpace()->recalculate_used_stable();
   }
   verify_work_stacks_empty();
   verify_overflow_empty();
@@ -5337,9 +5349,14 @@
     // further below.
     {
       CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());
+      
       // Update heap occupancy information which is used as
       // input to soft ref clearing policy at the next gc.
       Universe::update_heap_info_at_gc();
+   
+      // recalculate CMS used space after CMS collection
+      _cmsGen->cmsSpace()->recalculate_used_stable();
+ 
       _collectorState = Resizing;
     }
   }
@@ -5428,6 +5445,7 @@
     // Gather statistics on the young generation collection.
     collector()->stats().record_gc0_end(used());
   }
+  _cmsSpace->recalculate_used_stable();
 }
 
 void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* old_gen) {
--- old/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.hpp	Fri Jun 21 17:34:10 2019
+++ new/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.hpp	Fri Jun 21 17:34:09 2019
@@ -1112,6 +1112,7 @@
   double occupancy() const { return ((double)used())/((double)capacity()); }
   size_t contiguous_available() const;
   size_t unsafe_max_alloc_nogc() const;
+  size_t used_stable() const;
 
   // over-rides
   MemRegion used_region_at_save_marks() const;
--- old/src/hotspot/share/gc/cms/gSpaceCounters.hpp	Fri Jun 21 17:34:11 2019
+++ new/src/hotspot/share/gc/cms/gSpaceCounters.hpp	Fri Jun 21 17:34:11 2019
@@ -59,7 +59,7 @@
   }
 
   inline void update_used() {
-    _used->set_value(_gen->used());
+    _used->set_value(_gen->used_stable());
   }
 
   // special version of update_used() to allow the used value to be
@@ -103,7 +103,7 @@
     GenerationUsedHelper(Generation* g) : _gen(g) { }
 
     inline jlong take_sample() {
-      return _gen->used();
+      return _gen->used_stable();
     }
 };
 
--- old/src/hotspot/share/gc/shared/generation.cpp	Fri Jun 21 17:34:13 2019
+++ new/src/hotspot/share/gc/shared/generation.cpp	Fri Jun 21 17:34:12 2019
@@ -68,6 +68,12 @@
   return gch->old_gen_spec()->init_size();
 }
 
+// This is for CMS. It returns stable monotonic used space size.
+// Remove this when CMS is removed.
+size_t Generation::used_stable() const {
+  return used();
+}
+
 size_t Generation::max_capacity() const {
   return reserved().byte_size();
 }
--- old/src/hotspot/share/gc/shared/generation.hpp	Fri Jun 21 17:34:15 2019
+++ new/src/hotspot/share/gc/shared/generation.hpp	Fri Jun 21 17:34:14 2019
@@ -156,6 +156,7 @@
   virtual size_t capacity() const = 0;  // The maximum number of object bytes the
                                         // generation can currently hold.
   virtual size_t used() const = 0;      // The number of used bytes in the gen.
+  virtual size_t used_stable() const;   // The number of used bytes for memory monitoring tools.
   virtual size_t free() const = 0;      // The number of free bytes in the gen.
 
   // Support for java.lang.Runtime.maxMemory(); see CollectedHeap.