src/share/vm/gc_implementation/g1/g1Allocator.cpp

rev 7471 : 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
Summary: Evaluate and improve object copy time by micro-optimizations and splitting out slow and fast paths aggressively.
Reviewed-by:
Contributed-by: Tony Printezis <tprintezis@twitter.com>, Thomas Schatzl <thomas.schatzl@oracle.com>
rev 7472 : [mq]: 8060025-mikael-review1
rev 7473 : imported patch mikael-refactor-cset-state

*** 111,158 ****
  }
  
  G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size) :
    ParGCAllocBuffer(gclab_word_size), _retired(true) { }
  
! HeapWord* G1ParGCAllocator::allocate_slow(GCAllocPurpose purpose, size_t word_sz, AllocationContext_t context) {
!   HeapWord* obj = NULL;
!   size_t gclab_word_size = _g1h->desired_plab_sz(purpose);
    if (word_sz * 100 < gclab_word_size * ParallelGCBufferWastePct) {
!     G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose, context);
      add_to_alloc_buffer_waste(alloc_buf->words_remaining());
      alloc_buf->retire(false /* end_of_gc */, false /* retain */);
  
!     HeapWord* buf = _g1h->par_allocate_during_gc(purpose, gclab_word_size, context);
      if (buf == NULL) {
        return NULL; // Let caller handle allocation failure.
      }
      // Otherwise.
      alloc_buf->set_word_size(gclab_word_size);
      alloc_buf->set_buf(buf);
  
!     obj = alloc_buf->allocate(word_sz);
      assert(obj != NULL, "buffer was definitely big enough...");
    } else {
!     obj = _g1h->par_allocate_during_gc(purpose, word_sz, context);
    }
-   return obj;
  }
  
  G1DefaultParGCAllocator::G1DefaultParGCAllocator(G1CollectedHeap* g1h) :
              G1ParGCAllocator(g1h),
!             _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived)),
!             _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)) {
! 
!   _alloc_buffers[GCAllocForSurvived] = &_surviving_alloc_buffer;
!   _alloc_buffers[GCAllocForTenured]  = &_tenured_alloc_buffer;
! 
  }
  
  void G1DefaultParGCAllocator::retire_alloc_buffers() {
!   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
!     size_t waste = _alloc_buffers[ap]->words_remaining();
!     add_to_alloc_buffer_waste(waste);
!     _alloc_buffers[ap]->flush_stats_and_retire(_g1h->stats_for_purpose((GCAllocPurpose)ap),
                                                 true /* end_of_gc */,
                                                 false /* retain */);
    }
  }
--- 111,162 ----
  }
  
  G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size) :
    ParGCAllocBuffer(gclab_word_size), _retired(true) { }
  
! HeapWord* G1ParGCAllocator::allocate_direct_or_new_plab(InCSetState dest,
!                                                         size_t word_sz,
!                                                         AllocationContext_t context) {
!   size_t gclab_word_size = _g1h->desired_plab_sz(dest);
    if (word_sz * 100 < gclab_word_size * ParallelGCBufferWastePct) {
!     G1ParGCAllocBuffer* alloc_buf = alloc_buffer(dest, context);
      add_to_alloc_buffer_waste(alloc_buf->words_remaining());
      alloc_buf->retire(false /* end_of_gc */, false /* retain */);
  
!     HeapWord* buf = _g1h->par_allocate_during_gc(dest, gclab_word_size, context);
      if (buf == NULL) {
        return NULL; // Let caller handle allocation failure.
      }
      // Otherwise.
      alloc_buf->set_word_size(gclab_word_size);
      alloc_buf->set_buf(buf);
  
!     HeapWord* const obj = alloc_buf->allocate(word_sz);
      assert(obj != NULL, "buffer was definitely big enough...");
+     return obj;
    } else {
!     return _g1h->par_allocate_during_gc(dest, word_sz, context);
    }
  }
  
  G1DefaultParGCAllocator::G1DefaultParGCAllocator(G1CollectedHeap* g1h) :
    G1ParGCAllocator(g1h),
!   _surviving_alloc_buffer(g1h->desired_plab_sz(InCSetState::Young)),
!   _tenured_alloc_buffer(g1h->desired_plab_sz(InCSetState::Old)) {
!   for (uint state = 0; state < InCSetState::Num; state++) {
!     _alloc_buffers[state] = NULL;
!   }
!   _alloc_buffers[InCSetState::Young] = &_surviving_alloc_buffer;
!   _alloc_buffers[InCSetState::Old]  = &_tenured_alloc_buffer;
  }
  
  void G1DefaultParGCAllocator::retire_alloc_buffers() {
!   for (uint state = 0; state < InCSetState::Num; state++) {
!     G1ParGCAllocBuffer* const buf = _alloc_buffers[state];
!     if (buf != NULL) {
!       add_to_alloc_buffer_waste(buf->words_remaining());
!       buf->flush_stats_and_retire(_g1h->alloc_buffer_stats(state),
                                    true /* end_of_gc */,
                                    false /* retain */);
      }
+   }
  }