--- old/src/share/vm/gc/g1/g1AllocRegion.cpp 2015-08-19 10:56:38.880586584 +0200 +++ new/src/share/vm/gc/g1/g1AllocRegion.cpp 2015-08-19 10:56:38.800584228 +0200 @@ -205,11 +205,11 @@ } #if G1_ALLOC_REGION_TRACING -void G1AllocRegion::trace(const char* str, size_t word_size, HeapWord* result) { +void G1AllocRegion::trace(const char* str, size_t min_word_size, size_t desired_word_size, size_t actual_word_size, HeapWord* result) { // All the calls to trace that set either just the size or the size // and the result are considered part of level 2 tracing and are // skipped during level 1 tracing. - if ((word_size == 0 && result == NULL) || (G1_ALLOC_REGION_TRACING > 1)) { + if ((actual_word_size == 0 && result == NULL) || (G1_ALLOC_REGION_TRACING > 1)) { const size_t buffer_length = 128; char hr_buffer[buffer_length]; char rest_buffer[buffer_length]; @@ -226,10 +226,10 @@ if (G1_ALLOC_REGION_TRACING > 1) { if (result != NULL) { - jio_snprintf(rest_buffer, buffer_length, SIZE_FORMAT " " PTR_FORMAT, - word_size, result); - } else if (word_size != 0) { - jio_snprintf(rest_buffer, buffer_length, SIZE_FORMAT, word_size); + jio_snprintf(rest_buffer, buffer_length, "min " SIZE_FORMAT " desired " SIZE_FORMAT " actual " SIZE_FORMAT " " PTR_FORMAT, + min_word_size, desired_word_size, actual_word_size, result); + } else if (min_word_size != 0) { + jio_snprintf(rest_buffer, buffer_length, "min " SIZE_FORMAT " desired " SIZE_FORMAT, min_word_size, desired_word_size); } else { jio_snprintf(rest_buffer, buffer_length, ""); } --- old/src/share/vm/gc/g1/g1AllocRegion.hpp 2015-08-19 10:56:39.334599951 +0200 +++ new/src/share/vm/gc/g1/g1AllocRegion.hpp 2015-08-19 10:56:39.255597625 +0200 @@ -104,8 +104,13 @@ static inline HeapWord* par_allocate(HeapRegion* alloc_region, size_t word_size, bool bot_updates); + // Perform a MT-safe allocation out of the given region, with the given + // minimum and desired size. Returns the actual size allocated (between + // minimum and desired size) in actual_word_size if the allocation has been + // successful. static inline HeapWord* par_allocate(HeapRegion* alloc_region, size_t min_word_size, + size_t desired_word_size, size_t* actual_word_size, bool bot_updates); @@ -165,8 +170,15 @@ // or return NULL if it was unable to. inline HeapWord* attempt_allocation(size_t word_size, bool bot_updates); + // Perform an allocation out of the current allocation region, with the given + // minimum and desired size. Returns the actual size allocated (between + // minimum and desired size) in actual_word_size if the allocation has been + // successful. + // Should be called without holding a lock. It will try to allocate lock-free + // out of the active region, or return NULL if it was unable to. inline HeapWord* attempt_allocation(size_t min_word_size, - size_t* word_size, + size_t desired_word_size, + size_t* actual_word_size, bool bot_updates); // Second-level allocation: Should be called while holding a @@ -179,10 +191,11 @@ bool bot_updates); // Same as attempt_allocation_locked(size_t, bool), but allowing specification // of minimum word size of the block in min_word_size, and the maximum word - // size of the allocation in word_size. The actual size of the block is returned - // in word_size. + // size of the allocation in desired_word_size. The actual size of the block is + // returned in actual_word_size. inline HeapWord* attempt_allocation_locked(size_t min_word_size, - size_t* word_size, + size_t desired_word_size, + size_t* actual_word_size, bool bot_updates); // Should be called to allocate a new region even if the max of this @@ -206,9 +219,17 @@ virtual HeapRegion* release(); #if G1_ALLOC_REGION_TRACING - void trace(const char* str, size_t word_size = 0, HeapWord* result = NULL); + void trace(const char* str, + size_t min_word_size = 0, + size_t desired_word_size = 0, + size_t actual_word_size = 0, + HeapWord* result = NULL); #else // G1_ALLOC_REGION_TRACING - void trace(const char* str, size_t word_size = 0, HeapWord* result = NULL) { } + void trace(const char* str, + size_t min_word_size = 0, + size_t desired_word_size = 0, + size_t actual_word_size = 0, + HeapWord* result = NULL) { } #endif // G1_ALLOC_REGION_TRACING }; --- old/src/share/vm/gc/g1/g1AllocRegion.inline.hpp 2015-08-19 10:56:39.791613407 +0200 +++ new/src/share/vm/gc/g1/g1AllocRegion.inline.hpp 2015-08-19 10:56:39.709610992 +0200 @@ -41,66 +41,73 @@ } inline HeapWord* G1AllocRegion::par_allocate(HeapRegion* alloc_region, size_t word_size, bool bot_updates) { - return par_allocate(alloc_region, word_size, &word_size, bot_updates); + size_t temp; + return par_allocate(alloc_region, word_size, word_size, &temp, bot_updates); } inline HeapWord* G1AllocRegion::par_allocate(HeapRegion* alloc_region, size_t min_word_size, - size_t* word_size, + size_t desired_word_size, + size_t* actual_word_size, bool bot_updates) { assert(alloc_region != NULL, err_msg("pre-condition")); assert(!alloc_region->is_empty(), err_msg("pre-condition")); if (!bot_updates) { - return alloc_region->par_allocate_no_bot_updates(min_word_size, word_size); + return alloc_region->par_allocate_no_bot_updates(min_word_size, desired_word_size, actual_word_size); } else { - return alloc_region->par_allocate(min_word_size, word_size); + return alloc_region->par_allocate(min_word_size, desired_word_size, actual_word_size); } } inline HeapWord* G1AllocRegion::attempt_allocation(size_t word_size, bool bot_updates) { - return attempt_allocation(word_size, &word_size, bot_updates); + size_t temp; + return attempt_allocation(word_size, word_size, &temp, bot_updates); } inline HeapWord* G1AllocRegion::attempt_allocation(size_t min_word_size, - size_t* word_size, + size_t desired_word_size, + size_t* actual_word_size, bool bot_updates) { assert(bot_updates == _bot_updates, ar_ext_msg(this, "pre-condition")); HeapRegion* alloc_region = _alloc_region; assert(alloc_region != NULL, ar_ext_msg(this, "not initialized properly")); - HeapWord* result = par_allocate(alloc_region, min_word_size, word_size, bot_updates); + HeapWord* result = par_allocate(alloc_region, min_word_size, desired_word_size, actual_word_size, bot_updates); if (result != NULL) { - trace("alloc", *word_size, result); + trace("alloc", min_word_size, desired_word_size, *actual_word_size, result); return result; } - trace("alloc failed", *word_size); + trace("alloc failed", min_word_size, desired_word_size); return NULL; } inline HeapWord* G1AllocRegion::attempt_allocation_locked(size_t word_size, bool bot_updates) { - return attempt_allocation_locked(word_size, &word_size, bot_updates); + size_t temp; + return attempt_allocation_locked(word_size, word_size, &temp, bot_updates); } inline HeapWord* G1AllocRegion::attempt_allocation_locked(size_t min_word_size, - size_t* word_size, + size_t desired_word_size, + size_t* actual_word_size, bool bot_updates) { // First we have to redo the allocation, assuming we're holding the // appropriate lock, in case another thread changed the region while // we were waiting to get the lock. - HeapWord* result = attempt_allocation(min_word_size, word_size, bot_updates); + HeapWord* result = attempt_allocation(min_word_size, desired_word_size, actual_word_size, bot_updates); if (result != NULL) { return result; } retire(true /* fill_up */); - result = new_alloc_region_and_allocate(*word_size, false /* force */); + result = new_alloc_region_and_allocate(desired_word_size, false /* force */); if (result != NULL) { - trace("alloc locked (second attempt)", *word_size, result); + *actual_word_size = desired_word_size; + trace("alloc locked (second attempt)", min_word_size, desired_word_size, *actual_word_size, result); return result; } - trace("alloc locked failed", *word_size); + trace("alloc locked failed", min_word_size, desired_word_size); return NULL; } @@ -109,13 +116,13 @@ assert(bot_updates == _bot_updates, ar_ext_msg(this, "pre-condition")); assert(_alloc_region != NULL, ar_ext_msg(this, "not initialized properly")); - trace("forcing alloc"); + trace("forcing alloc", word_size, word_size); HeapWord* result = new_alloc_region_and_allocate(word_size, true /* force */); if (result != NULL) { - trace("alloc forced", word_size, result); + trace("alloc forced", word_size, word_size, word_size, result); return result; } - trace("alloc forced failed", word_size); + trace("alloc forced failed", word_size, word_size); return NULL; } --- old/src/share/vm/gc/g1/g1Allocator.cpp 2015-08-19 10:56:40.266627392 +0200 +++ new/src/share/vm/gc/g1/g1Allocator.cpp 2015-08-19 10:56:40.184624978 +0200 @@ -144,11 +144,20 @@ HeapWord* G1Allocator::par_allocate_during_gc(InCSetState dest, size_t word_size, AllocationContext_t context) { + size_t temp; + return par_allocate_during_gc(dest, word_size, word_size, &temp, context); +} + +HeapWord* G1Allocator::par_allocate_during_gc(InCSetState dest, + size_t min_word_size, + size_t desired_word_size, + size_t* actual_word_size, + AllocationContext_t context) { switch (dest.value()) { case InCSetState::Young: - return survivor_attempt_allocation(word_size, context); + return survivor_attempt_allocation(min_word_size, desired_word_size, actual_word_size, context); case InCSetState::Old: - return old_attempt_allocation(word_size, context); + return old_attempt_allocation(min_word_size, desired_word_size, actual_word_size, context); default: ShouldNotReachHere(); return NULL; // Keep some compilers happy @@ -171,37 +180,49 @@ _old_is_full = true; } -HeapWord* G1Allocator::survivor_attempt_allocation(size_t word_size, +HeapWord* G1Allocator::survivor_attempt_allocation(size_t min_word_size, + size_t desired_word_size, + size_t* actual_word_size, AllocationContext_t context) { - assert(!_g1h->is_humongous(word_size), + assert(!_g1h->is_humongous(desired_word_size), "we should not be seeing humongous-size allocations in this path"); - HeapWord* result = survivor_gc_alloc_region(context)->attempt_allocation(word_size, + HeapWord* result = survivor_gc_alloc_region(context)->attempt_allocation(min_word_size, + desired_word_size, + actual_word_size, false /* bot_updates */); if (result == NULL && !survivor_is_full(context)) { MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); - result = survivor_gc_alloc_region(context)->attempt_allocation_locked(word_size, + result = survivor_gc_alloc_region(context)->attempt_allocation_locked(min_word_size, + desired_word_size, + actual_word_size, false /* bot_updates */); if (result == NULL) { set_survivor_full(context); } } if (result != NULL) { - _g1h->dirty_young_block(result, word_size); + _g1h->dirty_young_block(result, *actual_word_size); } return result; } -HeapWord* G1Allocator::old_attempt_allocation(size_t word_size, +HeapWord* G1Allocator::old_attempt_allocation(size_t min_word_size, + size_t desired_word_size, + size_t* actual_word_size, AllocationContext_t context) { - assert(!_g1h->is_humongous(word_size), + assert(!_g1h->is_humongous(desired_word_size), "we should not be seeing humongous-size allocations in this path"); - HeapWord* result = old_gc_alloc_region(context)->attempt_allocation(word_size, + HeapWord* result = old_gc_alloc_region(context)->attempt_allocation(min_word_size, + desired_word_size, + actual_word_size, true /* bot_updates */); if (result == NULL && !old_is_full(context)) { MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); - result = old_gc_alloc_region(context)->attempt_allocation_locked(word_size, + result = old_gc_alloc_region(context)->attempt_allocation_locked(min_word_size, + desired_word_size, + actual_word_size, true /* bot_updates */); if (result == NULL) { set_old_full(context); @@ -243,10 +264,15 @@ G1PLAB* alloc_buf = alloc_buffer(dest, context); alloc_buf->retire(); - HeapWord* buf = _allocator->par_allocate_during_gc(dest, plab_word_size, context); + size_t actual_plab_size; + HeapWord* buf = _allocator->par_allocate_during_gc(dest, + required_in_plab, + plab_word_size, + &actual_plab_size, + context); if (buf != NULL) { // Otherwise. - alloc_buf->set_buf(buf, plab_word_size); + alloc_buf->set_buf(buf, actual_plab_size); HeapWord* const obj = alloc_buf->allocate(word_sz); assert(obj != NULL, err_msg("PLAB should have been big enough, tried to allocate " --- old/src/share/vm/gc/g1/g1Allocator.hpp 2015-08-19 10:56:40.728640995 +0200 +++ new/src/share/vm/gc/g1/g1Allocator.hpp 2015-08-19 10:56:40.648638639 +0200 @@ -57,10 +57,14 @@ virtual OldGCAllocRegion* old_gc_alloc_region(AllocationContext_t context) = 0; // Allocation attempt during GC for a survivor object / PLAB. - inline HeapWord* survivor_attempt_allocation(size_t word_size, + inline HeapWord* survivor_attempt_allocation(size_t min_word_size, + size_t desired_word_size, + size_t* actual_word_size, AllocationContext_t context); // Allocation attempt during GC for an old object / PLAB. - inline HeapWord* old_attempt_allocation(size_t word_size, + inline HeapWord* old_attempt_allocation(size_t min_word_size, + size_t desired_word_size, + size_t* actual_word_size, AllocationContext_t context); public: G1Allocator(G1CollectedHeap* heap) : _g1h(heap), _survivor_is_full(false), _old_is_full(false) { } @@ -102,6 +106,12 @@ size_t word_size, AllocationContext_t context); + HeapWord* par_allocate_during_gc(InCSetState dest, + size_t min_word_size, + size_t desired_word_size, + size_t* actual_word_size, + AllocationContext_t context); + virtual size_t used_in_alloc_regions() = 0; }; --- old/src/share/vm/gc/g1/heapRegion.hpp 2015-08-19 10:56:41.186654480 +0200 +++ new/src/share/vm/gc/g1/heapRegion.hpp 2015-08-19 10:56:41.106652124 +0200 @@ -109,7 +109,7 @@ // evacuation pauses between two cleanups, which is _highly_ unlikely. class G1OffsetTableContigSpace: public CompactibleSpace { friend class VMStructs; - HeapWord* _top; + HeapWord* volatile _top; HeapWord* volatile _scan_top; protected: G1BlockOffsetArrayContigSpace _offsets; @@ -134,10 +134,18 @@ // Reset the G1OffsetTableContigSpace. virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); - HeapWord** top_addr() { return &_top; } - // Allocation helpers (return NULL if full). - inline HeapWord* allocate_impl(size_t min_word_size, size_t* actual_word_size, HeapWord* end_value); - inline HeapWord* par_allocate_impl(size_t min_word_size, size_t* actual_word_size, HeapWord* end_value); + HeapWord* volatile* top_addr() { return &_top; } + // Try to allocate at least min_word_size and up to desired_size from this Space. + // Returns NULL if not possible, otherwise sets actual_word_size to the amount of + // space allocated. + // This version assumes that all allocation requests to this Space are properly + // synchronized. + inline HeapWord* allocate_impl(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size); + // Try to allocate at least min_word_size and up to desired_size from this Space. + // Returns NULL if not possible, otherwise sets actual_word_size to the amount of + // space allocated. + // This version synchronizes with other calls to par_allocate_impl(). + inline HeapWord* par_allocate_impl(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size); public: void reset_after_compaction() { set_top(compaction_top()); } @@ -181,9 +189,9 @@ // Allocation (return NULL if full). Assumes the caller has established // mutually exclusive access to the space. - HeapWord* allocate(size_t min_word_size, size_t* word_size); + HeapWord* allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size); // Allocation (return NULL if full). Enforces mutual exclusion internally. - HeapWord* par_allocate(size_t min_word_size, size_t* word_size); + HeapWord* par_allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size); virtual HeapWord* allocate(size_t word_size); virtual HeapWord* par_allocate(size_t word_size); @@ -356,9 +364,9 @@ // Override for scan_and_forward support. void prepare_for_compaction(CompactPoint* cp); - inline HeapWord* par_allocate_no_bot_updates(size_t min_word_size, size_t* word_size); + inline HeapWord* par_allocate_no_bot_updates(size_t min_word_size, size_t desired_word_size, size_t* word_size); inline HeapWord* allocate_no_bot_updates(size_t word_size); - inline HeapWord* allocate_no_bot_updates(size_t min_word_size, size_t* word_size); + inline HeapWord* allocate_no_bot_updates(size_t min_word_size, size_t desired_word_size, size_t* actual_size); // If this region is a member of a HeapRegionManager, the index in that // sequence, otherwise -1. --- old/src/share/vm/gc/g1/heapRegion.inline.hpp 2015-08-19 10:56:41.653668230 +0200 +++ new/src/share/vm/gc/g1/heapRegion.inline.hpp 2015-08-19 10:56:41.574665904 +0200 @@ -32,32 +32,30 @@ #include "oops/oop.inline.hpp" #include "runtime/atomic.inline.hpp" -// This version requires locking. inline HeapWord* G1OffsetTableContigSpace::allocate_impl(size_t min_word_size, - size_t* actual_size, - HeapWord* const end_value) { + size_t desired_word_size, + size_t* actual_size) { HeapWord* obj = top(); - size_t available = pointer_delta(end_value, obj); - size_t want_to_allocate = MIN2(available, *actual_size); + size_t available = pointer_delta(end(), obj); + size_t want_to_allocate = MIN2(available, desired_word_size); if (want_to_allocate >= min_word_size) { HeapWord* new_top = obj + want_to_allocate; set_top(new_top); assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); *actual_size = want_to_allocate; - return obj; + return obj; } else { return NULL; } } -// This version is lock-free. inline HeapWord* G1OffsetTableContigSpace::par_allocate_impl(size_t min_word_size, - size_t* actual_size, - HeapWord* const end_value) { + size_t desired_word_size, + size_t* actual_size) { do { HeapWord* obj = top(); - size_t available = pointer_delta(end_value, obj); - size_t want_to_allocate = MIN2(available, *actual_size); + size_t available = pointer_delta(end(), obj); + size_t want_to_allocate = MIN2(available, desired_word_size); if (want_to_allocate >= min_word_size) { HeapWord* new_top = obj + want_to_allocate; HeapWord* result = (HeapWord*)Atomic::cmpxchg_ptr(new_top, top_addr(), obj); @@ -75,23 +73,34 @@ } while (true); } -inline HeapWord* G1OffsetTableContigSpace::allocate(size_t min_word_size, size_t* actual_size) { - HeapWord* res = allocate_impl(min_word_size, actual_size, end()); +inline HeapWord* G1OffsetTableContigSpace::allocate(size_t min_word_size, + size_t desired_word_size, + size_t* actual_size) { + HeapWord* res = allocate_impl(min_word_size, desired_word_size, actual_size); if (res != NULL) { _offsets.alloc_block(res, *actual_size); } return res; } -inline HeapWord* G1OffsetTableContigSpace::allocate(size_t word_size) { return allocate(word_size, &word_size); } -inline HeapWord* G1OffsetTableContigSpace::par_allocate(size_t word_size) { return par_allocate(word_size, &word_size); } +inline HeapWord* G1OffsetTableContigSpace::allocate(size_t word_size) { + size_t temp; + return allocate(word_size, word_size, &temp); +} + +inline HeapWord* G1OffsetTableContigSpace::par_allocate(size_t word_size) { + size_t temp; + return par_allocate(word_size, word_size, &temp); +} // Because of the requirement of keeping "_offsets" up to date with the // allocations, we sequentialize these with a lock. Therefore, best if // this is used for larger LAB allocations only. -inline HeapWord* G1OffsetTableContigSpace::par_allocate(size_t min_word_size, size_t* actual_size) { +inline HeapWord* G1OffsetTableContigSpace::par_allocate(size_t min_word_size, + size_t desired_word_size, + size_t* actual_size) { MutexLocker x(&_par_alloc_lock); - return allocate(min_word_size, actual_size); + return allocate(min_word_size, desired_word_size, actual_size); } inline HeapWord* G1OffsetTableContigSpace::block_start(const void* p) { @@ -139,18 +148,23 @@ return pointer_delta(next, addr); } -inline HeapWord* HeapRegion::par_allocate_no_bot_updates(size_t min_word_size, size_t* actual_word_size) { +inline HeapWord* HeapRegion::par_allocate_no_bot_updates(size_t min_word_size, + size_t desired_word_size, + size_t* actual_word_size) { assert(is_young(), "we can only skip BOT updates on young regions"); - return par_allocate_impl(min_word_size, actual_word_size, end()); + return par_allocate_impl(min_word_size, desired_word_size, actual_word_size); } inline HeapWord* HeapRegion::allocate_no_bot_updates(size_t word_size) { - return allocate_no_bot_updates(word_size, &word_size); + size_t temp; + return allocate_no_bot_updates(word_size, word_size, &temp); } -inline HeapWord* HeapRegion::allocate_no_bot_updates(size_t min_word_size, size_t* actual_word_size) { +inline HeapWord* HeapRegion::allocate_no_bot_updates(size_t min_word_size, + size_t desired_word_size, + size_t* actual_word_size) { assert(is_young(), "we can only skip BOT updates on young regions"); - return allocate_impl(min_word_size, actual_word_size, end()); + return allocate_impl(min_word_size, desired_word_size, actual_word_size); } inline void HeapRegion::note_start_of_marking() { --- old/src/share/vm/gc/g1/vmStructs_g1.hpp 2015-08-19 10:56:42.109681656 +0200 +++ new/src/share/vm/gc/g1/vmStructs_g1.hpp 2015-08-19 10:56:42.030679330 +0200 @@ -34,7 +34,7 @@ static_field(HeapRegion, GrainBytes, size_t) \ static_field(HeapRegion, LogOfHRGrainBytes, int) \ \ - nonstatic_field(G1OffsetTableContigSpace, _top, HeapWord*) \ + nonstatic_field(G1OffsetTableContigSpace, _top, HeapWord* volatile) \ \ nonstatic_field(G1HeapRegionTable, _base, address) \ nonstatic_field(G1HeapRegionTable, _length, size_t) \