src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp

rev 47223 : [mq]: heapz8
rev 47224 : [mq]: heap9a

*** 172,183 ****
--- 172,196 ----
                                    HeapWord* top,
                                    size_t    new_size) {
    _number_of_refills++;
    print_stats("fill");
    assert(top <= start + new_size - alignment_reserve(), "size too small");
+ 
+   // Remember old bytes until sample for the next tlab only if this is our first
+   // actual refill.
+   size_t old_bytes_until_sample = 0;
+   if (_number_of_refills > 1) {
+     old_bytes_until_sample = bytes_until_sample();
+   }
+ 
    initialize(start, top, start + new_size - alignment_reserve());
  
+   if (old_bytes_until_sample > 0) {
+     set_bytes_until_sample(old_bytes_until_sample);
+     set_sample_end();
+   }
+ 
    // Reset amount of internal fragmentation
    set_refill_waste_limit(initial_refill_waste_limit());
  }
  
  void ThreadLocalAllocBuffer::initialize(HeapWord* start,
*** 311,330 ****
      p += oop(p)->size();
    }
    guarantee(p == top(), "end of last object must match end of space");
  }
  
! void ThreadLocalAllocBuffer::pick_next_sample() {
!   if (!HeapMonitoring::enabled()) {
!     return;
!   }
! 
!   if (bytes_until_sample() == 0) {
!     HeapMonitoring::pick_next_sample(bytes_until_sample_addr());
!   }
! 
!   // Finally, fix up the sampling bytes left and _end.
    size_t heap_words_remaining = _end - _top;
    size_t bytes_left = bytes_until_sample();
    size_t words_until_sample = bytes_left / HeapWordSize;
  
    if (heap_words_remaining > words_until_sample) {
--- 324,334 ----
      p += oop(p)->size();
    }
    guarantee(p == top(), "end of last object must match end of space");
  }
  
! void ThreadLocalAllocBuffer::set_sample_end() {
    size_t heap_words_remaining = _end - _top;
    size_t bytes_left = bytes_until_sample();
    size_t words_until_sample = bytes_left / HeapWordSize;
  
    if (heap_words_remaining > words_until_sample) {
*** 334,343 ****
--- 338,366 ----
      set_bytes_until_sample(0);
    } else {
      bytes_left -= heap_words_remaining * HeapWordSize;
      set_bytes_until_sample(bytes_left);
    }
+ }
+ 
+ void ThreadLocalAllocBuffer::pick_next_sample(size_t diff) {
+   if (!HeapMonitoring::enabled()) {
+     return;
+   }
+ 
+   if (bytes_until_sample() == 0) {
+     HeapMonitoring::pick_next_sample(bytes_until_sample_addr());
+   }
+ 
+   if (diff > 0) {
+     // Try to correct sample size by removing extra space from last allocation.
+     if (bytes_until_sample() > diff * HeapWordSize) {
+       set_bytes_until_sample(bytes_until_sample() - diff * HeapWordSize);
+     }
+   }
+ 
+   set_sample_end();
  
    log_trace(gc, tlab)("TLAB picked next sample: thread: " INTPTR_FORMAT " [id: %2d]"
                        " start: %p  top: %p end: %p actual_end: %p slow_path_end: %p",
                        p2i(myThread()), myThread()->osthread()->thread_id(),
                        start(), top(), end(),
*** 365,384 ****
                                             size_t size) {
    if (!HeapMonitoring::enabled()) {
      return;
    }
  
!   set_bytes_until_sample(bytes_until_sample() - size);
  
    // Should we sample now?
-   set_back_actual_end();
    if (should_sample()) {
      HeapMonitoring::object_alloc_do_sample(thread,
                                             reinterpret_cast<oopDesc*>(result),
!                                            size);
!   }
    pick_next_sample();
  }
  
  HeapWord* ThreadLocalAllocBuffer::hard_end() {
    // Did a fast TLAB refill occur?
    if (_slow_path_end != _end) {
--- 388,414 ----
                                             size_t size) {
    if (!HeapMonitoring::enabled()) {
      return;
    }
  
!   size_t size_in_bytes = size * HeapWordSize;
!   if (bytes_until_sample() > size_in_bytes) {
!     set_bytes_until_sample(bytes_until_sample() - size_in_bytes);
!   } else {
!     // Technically this is not exactly right, we probably should remember how many bytes are
!     // negative probably to then reduce our next sample size.
!     set_bytes_until_sample(0);
!   }
  
    // Should we sample now?
    if (should_sample()) {
      HeapMonitoring::object_alloc_do_sample(thread,
                                             reinterpret_cast<oopDesc*>(result),
!                                            size_in_bytes);
!     set_back_actual_end();
      pick_next_sample();
+   }
  }
  
  HeapWord* ThreadLocalAllocBuffer::hard_end() {
    // Did a fast TLAB refill occur?
    if (_slow_path_end != _end) {