--- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2013-02-04 11:23:27.581040285 -0800 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2013-02-04 11:23:27.220860295 -0800 @@ -4364,8 +4364,6 @@ _g1_rem(g1h->g1_rem_set()), _hash_seed(17), _queue_num(queue_num), _term_attempts(0), - _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived)), - _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)), _age_table(false), _strong_roots_time(0), _term_time(0), _alloc_buffer_waste(0), _undo_waste(0) { @@ -4385,8 +4383,12 @@ _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM; memset(_surviving_young_words, 0, (size_t) real_length * sizeof(size_t)); - _alloc_buffers[GCAllocForSurvived] = &_surviving_alloc_buffer; - _alloc_buffers[GCAllocForTenured] = &_tenured_alloc_buffer; + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + for (int pr = 0; pr < GCAllocPriorityCount; ++pr) { + _alloc_buffers[buf_idx(ap, pr)] = + new G1ParGCAllocBuffer(g1h->desired_plab_sz(GCAllocPurpose(ap))); + } + } _start = os::elapsedTime(); } --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2013-02-04 11:23:29.373936238 -0800 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2013-02-04 11:23:28.777638254 -0800 @@ -590,11 +590,6 @@ // may not be a humongous - it must fit into a single heap region. HeapWord* par_allocate_during_gc(GCAllocPurpose purpose, size_t word_size); - HeapWord* allocate_during_gc_slow(GCAllocPurpose purpose, - HeapRegion* alloc_region, - bool par, - size_t word_size); - // Ensure that no further allocations can happen in "r", bearing in mind // that parallel threads might be attempting allocations. void par_allocate_remaining_space(HeapRegion* r); @@ -1746,6 +1741,17 @@ }; class G1ParScanThreadState : public StackObj { +private: + enum GCAllocPriority { + GCAllocPriority1, + GCAllocPriority2, + GCAllocPriorityCount + }; + + int buf_idx(int purpose, int priority) { + return purpose*GCAllocPriorityCount + priority; + } + protected: G1CollectedHeap* _g1h; RefToScanQueue* _refs; @@ -1753,9 +1759,7 @@ CardTableModRefBS* _ct_bs; G1RemSet* _g1_rem; - G1ParGCAllocBuffer _surviving_alloc_buffer; - G1ParGCAllocBuffer _tenured_alloc_buffer; - G1ParGCAllocBuffer* _alloc_buffers[GCAllocPurposeCount]; + G1ParGCAllocBuffer* _alloc_buffers[GCAllocPurposeCount*GCAllocPriorityCount]; ageTable _age_table; size_t _alloc_buffer_waste; @@ -1814,13 +1818,18 @@ ~G1ParScanThreadState() { FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base, mtGC); + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + for (int pr = 0; pr < GCAllocPriorityCount; ++pr) { + delete alloc_buffer(GCAllocPurpose(ap), GCAllocPriority(pr)); + } + } } RefToScanQueue* refs() { return _refs; } ageTable* age_table() { return &_age_table; } - G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) { - return _alloc_buffers[purpose]; + G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose, GCAllocPriority priority) { + return _alloc_buffers[buf_idx(purpose, priority)]; } size_t alloc_buffer_waste() const { return _alloc_buffer_waste; } @@ -1849,18 +1858,24 @@ HeapWord* obj = NULL; size_t gclab_word_size = _g1h->desired_plab_sz(purpose); if (word_sz * 100 < gclab_word_size * ParallelGCBufferWastePct) { - G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose); - add_to_alloc_buffer_waste(alloc_buf->words_remaining()); - alloc_buf->retire(false /* end_of_gc */, false /* retain */); + G1ParGCAllocBuffer* alloc_buf1 = alloc_buffer(purpose, GCAllocPriority1); + G1ParGCAllocBuffer* alloc_buf2 = alloc_buffer(purpose, GCAllocPriority2); + + add_to_alloc_buffer_waste(alloc_buf1->words_remaining()); + alloc_buf1->retire(false /* end_of_gc */, false /* retain */); HeapWord* buf = _g1h->par_allocate_during_gc(purpose, gclab_word_size); if (buf == NULL) return NULL; // Let caller handle allocation failure. // Otherwise. - alloc_buf->set_word_size(gclab_word_size); - alloc_buf->set_buf(buf); + alloc_buf1->set_word_size(gclab_word_size); + alloc_buf1->set_buf(buf); - obj = alloc_buf->allocate(word_sz); + obj = alloc_buf1->allocate(word_sz); assert(obj != NULL, "buffer was definitely big enough..."); + + // Swap buffers in order to preserve the priority order + _alloc_buffers[buf_idx(purpose, GCAllocPriority1)] = alloc_buf2; + _alloc_buffers[buf_idx(purpose, GCAllocPriority2)] = alloc_buf1; } else { obj = _g1h->par_allocate_during_gc(purpose, word_sz); } @@ -1868,20 +1883,27 @@ } HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) { - HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz); - if (obj != NULL) return obj; + HeapWord* obj; + for (int pr = 0; pr < GCAllocPriorityCount; ++pr) { + obj = alloc_buffer(purpose, (GCAllocPriority)pr)->allocate(word_sz); + if (obj != NULL) return obj; + } + // Otherwise. return allocate_slow(purpose, word_sz); } void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) { - if (alloc_buffer(purpose)->contains(obj)) { - assert(alloc_buffer(purpose)->contains(obj + word_sz - 1), - "should contain whole object"); - alloc_buffer(purpose)->undo_allocation(obj, word_sz); - } else { - CollectedHeap::fill_with_object(obj, word_sz); - add_to_undo_waste(word_sz); + for (int pr = 0; pr < GCAllocPriorityCount; ++pr) { + if (alloc_buffer(purpose, (GCAllocPriority)pr)->contains(obj)) { + assert(alloc_buffer(purpose, (GCAllocPriority)pr)->contains(obj + word_sz - 1), + "should contain whole object"); + alloc_buffer(purpose, (GCAllocPriority)pr)->undo_allocation(obj, word_sz); + return; + } } + + CollectedHeap::fill_with_object(obj, word_sz); + add_to_undo_waste(word_sz); } void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) { @@ -1939,11 +1961,14 @@ void retire_alloc_buffers() { for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { - size_t waste = _alloc_buffers[ap]->words_remaining(); - add_to_alloc_buffer_waste(waste); - _alloc_buffers[ap]->flush_stats_and_retire(_g1h->stats_for_purpose((GCAllocPurpose)ap), - true /* end_of_gc */, - false /* retain */); + for (int pr = 0; pr < GCAllocPriorityCount; ++pr) { + G1ParGCAllocBuffer* buffer = alloc_buffer((GCAllocPurpose)ap, (GCAllocPriority)pr); + size_t waste = buffer->words_remaining(); + add_to_alloc_buffer_waste(waste); + buffer->flush_stats_and_retire(_g1h->stats_for_purpose((GCAllocPurpose)ap), + true /* end_of_gc */, + false /* retain */); + } } } @@ -1969,7 +1994,6 @@ } } -public: void trim_queue(); };