src/hotspot/share/gc/shared/collectedHeap.cpp

rev 48551 : [mq]: heap8
rev 48552 : [mq]: heap10a
rev 48553 : [mq]: heap14_rebased
rev 48557 : [mq]: heap17
rev 48562 : [mq]: heap23

@@ -36,10 +36,11 @@
 #include "logging/log.hpp"
 #include "memory/metaspace.hpp"
 #include "memory/resourceArea.hpp"
 #include "oops/instanceMirrorKlass.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/heapMonitoring.hpp"
 #include "runtime/init.hpp"
 #include "runtime/thread.inline.hpp"
 #include "runtime/threadSMR.hpp"
 #include "services/heapDumper.hpp"
 #include "utilities/align.hpp"

@@ -287,11 +288,48 @@
     thread->check_for_valid_safepoint_state(true);
   }
 }
 #endif
 
+
+void CollectedHeap::sample_allocation(Thread* thread, HeapWord* obj,
+                                      size_t size, size_t overflowed_words) {
+  // Object is allocated, sample it now.
+  HeapMonitoring::object_alloc_do_sample(thread,
+                                         reinterpret_cast<oopDesc*>(obj),
+                                         size * HeapWordSize);
+  // Pick a next sample in this case, we allocated right.
+  thread->tlab().pick_next_sample(overflowed_words);
+}
+
+HeapWord* CollectedHeap::allocate_sampled_object(Thread* thread, size_t size) {
+  thread->tlab().set_back_allocation_end();
+
+  // The tlab could still have space after this sample.
+  return thread->tlab().allocate(size);
+}
+
 HeapWord* CollectedHeap::allocate_from_tlab_slow(Klass* klass, Thread* thread, size_t size) {
+  // In case the tlab changes, remember if this one wanted a sample.
+  bool should_sample = HeapMonitoring::enabled() && thread->tlab().should_sample();
+
+  HeapWord* obj = NULL;
+  if (should_sample) {
+    // Remember the tlab end to fix up the sampling rate.
+    HeapWord* tlab_old_end = thread->tlab().current_end();
+    obj = allocate_sampled_object(thread, size);
+
+    // If we did allocate in this tlab, sample it. Otherwise, we wait for the
+    // new tlab's first allocation at the end of this method.
+    if (obj != NULL) {
+      // Fix sample rate by removing the extra words allocated in this last
+      // sample.
+      size_t overflowed_words = pointer_delta(thread->tlab().top(), tlab_old_end);
+      sample_allocation(thread, obj, size, overflowed_words);
+      return obj;
+    }
+  }
 
   // Retain tlab and allocate object in shared space if
   // the amount free in the tlab is too large to discard.
   if (thread->tlab().free() > thread->tlab().refill_waste_limit()) {
     thread->tlab().record_slow_allocation(size);

@@ -307,11 +345,11 @@
   if (new_tlab_size == 0) {
     return NULL;
   }
 
   // Allocate a new TLAB...
-  HeapWord* obj = Universe::heap()->allocate_new_tlab(new_tlab_size);
+  obj = Universe::heap()->allocate_new_tlab(new_tlab_size);
   if (obj == NULL) {
     return NULL;
   }
 
   AllocTracer::send_allocation_in_new_tlab(klass, obj, new_tlab_size * HeapWordSize, size * HeapWordSize, thread);

@@ -328,10 +366,15 @@
     size_t hdr_size = oopDesc::header_size();
     Copy::fill_to_words(obj + hdr_size, new_tlab_size - hdr_size, badHeapWordVal);
 #endif // ASSERT
   }
   thread->tlab().fill(obj, obj + size, new_tlab_size);
+
+  // Did we initially want to sample?
+  if (should_sample) {
+    sample_allocation(thread, obj, size);
+  }
   return obj;
 }
 
 void CollectedHeap::flush_deferred_store_barrier(JavaThread* thread) {
   MemRegion deferred = thread->deferred_card_mark();