--- old/src/share/vm/gc_implementation/g1/concurrentMark.cpp 2012-09-20 11:25:41.701263798 -0700 +++ new/src/share/vm/gc_implementation/g1/concurrentMark.cpp 2012-09-20 11:25:41.477928547 -0700 @@ -47,7 +47,7 @@ // Concurrent marking bit map wrapper CMBitMapRO::CMBitMapRO(ReservedSpace rs, int shifter) : - _bm((uintptr_t*)NULL,0), + _bm((uintptr_t*)NULL, 0), _shifter(shifter) { _bmStartWord = (HeapWord*)(rs.base()); _bmWordSize = rs.size()/HeapWordSize; // rs.size() is in bytes @@ -164,10 +164,23 @@ {} void CMMarkStack::allocate(size_t size) { - _base = NEW_C_HEAP_ARRAY(oop, size, mtGC); - if (_base == NULL) { - vm_exit_during_initialization("Failed to allocate CM region mark stack"); - } + size_t mark_stack_bytes = size * oopSize; + size_t aligned_mark_stack_bytes = + ReservedSpace::allocation_align_size_up(mark_stack_bytes); + + ReservedSpace ms_rs(aligned_mark_stack_bytes); + if (!ms_rs.is_reserved()) { + vm_exit_during_initialization("Failed to allocate CM mark stack"); + } + if (!_virtual_space.initialize(ms_rs, ms_rs.size())) { + vm_exit_during_initialization("Couldn't reseve backing store " + "for CM mark stack"); + } + guarantee(_virtual_space.committed_size() == ms_rs.size(), + "Didn't reserve backing store for " + "all of concurrent marking stack?"); + + _base = (oop*) _virtual_space.low(); _index = 0; _capacity = (jint) size; _saved_index = -1; @@ -176,7 +189,8 @@ CMMarkStack::~CMMarkStack() { if (_base != NULL) { - FREE_C_HEAP_ARRAY(oop, _base, mtGC); + _base = NULL; + _virtual_space.release(); } } @@ -404,9 +418,9 @@ return MAX2((n_par_threads + 2) / 4, 1U); } -ConcurrentMark::ConcurrentMark(ReservedSpace rs, uint max_regions) : - _markBitMap1(rs, MinObjAlignment - 1), - _markBitMap2(rs, MinObjAlignment - 1), +ConcurrentMark::ConcurrentMark(ReservedSpace heap_rs, uint max_regions) : + _markBitMap1(heap_rs, MinObjAlignment - 1), + _markBitMap2(heap_rs, MinObjAlignment - 1), _parallel_marking_threads(0), _max_parallel_marking_threads(0), @@ -415,14 +429,12 @@ _cleanup_sleep_factor(0.0), _cleanup_task_overhead(1.0), _cleanup_list("Cleanup List"), - _region_bm((BitMap::idx_t) max_regions, false /* in_resource_area*/), - _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >> - CardTableModRefBS::card_shift, - false /* in_resource_area*/), _prevMarkBitMap(&_markBitMap1), _nextMarkBitMap(&_markBitMap2), + _region_bm((BitMap::idx_t) max_regions, false /* in_resource_area*/), + _markStack(this), // _finger set in set_non_marking_state @@ -473,31 +485,83 @@ _g1h = G1CollectedHeap::heap(); assert(CGC_lock != NULL, "Where's the CGC_lock?"); - assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency"); - assert(_markBitMap2.covers(rs), "_markBitMap2 inconsistency"); + assert(_markBitMap1.covers(heap_rs), "_markBitMap1 inconsistency"); + assert(_markBitMap2.covers(heap_rs), "_markBitMap2 inconsistency"); SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); satb_qs.set_buffer_size(G1SATBBufferSize); _root_regions.init(_g1h, this); + // Calculate the size of the memory that is needed for the backing store + // (from which we "allocate" the global liveness card bitmap and the + // liveness card bitmaps for the individual worker threads). + + // Card liveness bitmap size (in bits) + BitMap::idx_t card_bm_size = (heap_rs.size() + CardTableModRefBS::card_size - 1) + >> CardTableModRefBS::card_shift; + // Card liveness bitmap size (in bytes) + size_t card_bm_size_bytes = (card_bm_size + (BitsPerByte - 1)) / BitsPerByte; + + // Length of marked bytes array (one element per region) + size_t marked_bytes_size = max_regions; + + // Size of marked bytes array (in bytes) + size_t marked_bytes_size_bytes = marked_bytes_size * sizeof(size_t); + + // Now align up to whole number of bitmap words + assert(sizeof(BitMap::bm_word_t) >= sizeof(size_t), "check this code"); + card_bm_size_bytes = align_size_up(card_bm_size_bytes, sizeof(BitMap::bm_word_t)); + marked_bytes_size_bytes = align_size_up(marked_bytes_size_bytes, sizeof(BitMap::bm_word_t)); + + // The memory size is the size of the global card bitmap, + // and the combined size of the card bitmaps and marked bytes + // arrays for each of the worker threads. + size_t mem_size_bytes = card_bm_size_bytes + + _max_task_num * (card_bm_size_bytes + marked_bytes_size_bytes); + + // Align the size up. + size_t aligned_mem_size_bytes = ReservedSpace::allocation_align_size_up(mem_size_bytes); + + // Reserve the backing store + ReservedSpace cm_rs(aligned_mem_size_bytes); + if (!cm_rs.is_reserved()) { + vm_exit_during_initialization("Failed to allocate CM backing store"); + } + if (!_virtual_space.initialize(cm_rs, cm_rs.size())) { + vm_exit_during_initialization("Couldn't reseve CM backing store"); + } + guarantee(_virtual_space.committed_size() == cm_rs.size(), + "didn't reserve all of CM backing store?"); + guarantee(_virtual_space.committed_size() >= mem_size_bytes, "sanity"); + _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num, mtGC); _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num, mtGC); _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_task_num, mtGC); _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_task_num, mtGC); - BitMap::idx_t card_bm_size = _card_bm.size(); - // so that the assertion in MarkingTaskQueue::task_queue doesn't fail _active_tasks = _max_task_num; + + char* vaddr = _virtual_space.low(); + size_t incr_bytes = card_bm_size_bytes + marked_bytes_size_bytes; + for (int i = 0; i < (int) _max_task_num; ++i) { CMTaskQueue* task_queue = new CMTaskQueue(); task_queue->initialize(); _task_queues->register_queue(i, task_queue); + + // "allocate" the card bitmap and live bytes array for current task. + guarantee(_virtual_space.low() <= vaddr && vaddr < _virtual_space.high(), "oob"); + guarantee((vaddr + incr_bytes) < _virtual_space.high(), "overflow"); + + char* card_bm_map_addr = vaddr; + _count_card_bitmaps[i].set_map((BitMap::bm_word_t*) card_bm_map_addr); + _count_card_bitmaps[i].set_size((BitMap::idx_t) card_bm_size); - _count_card_bitmaps[i] = BitMap(card_bm_size, false); - _count_marked_bytes[i] = NEW_C_HEAP_ARRAY(size_t, (size_t) max_regions, mtGC); + char* marked_bytes_addr = card_bm_map_addr + card_bm_size_bytes; + _count_marked_bytes[i] = (size_t*) marked_bytes_addr; _tasks[i] = new CMTask(i, this, _count_marked_bytes[i], @@ -505,8 +569,17 @@ task_queue, _task_queues); _accum_task_vtime[i] = 0.0; + + // Update pointer into virtual memory backing store + vaddr += incr_bytes; } + // Now "allocate" the global card bitmap. + guarantee(_virtual_space.low() <= vaddr && vaddr < _virtual_space.high(), "oob"); + guarantee((vaddr + card_bm_size_bytes) <= _virtual_space.high(), "overflow"); + _card_bm.set_map((BitMap::bm_word_t*) vaddr); + _card_bm.set_size((BitMap::idx_t) card_bm_size); + // Calculate the card number for the bottom of the heap. Used // in biasing indexes into the accounting card bitmaps. _heap_bottom_card_num = @@ -591,7 +664,7 @@ } // so that the call below can read a sensible value - _heap_start = (HeapWord*) rs.base(); + _heap_start = (HeapWord*) heap_rs.base(); set_non_marking_state(); } --- old/src/share/vm/gc_implementation/g1/concurrentMark.hpp 2012-09-20 11:25:43.326693349 -0700 +++ new/src/share/vm/gc_implementation/g1/concurrentMark.hpp 2012-09-20 11:25:43.108756790 -0700 @@ -164,6 +164,7 @@ jint _capacity; // max #elements jint _saved_index; // value of _index saved at start of GC NOT_PRODUCT(jint _max_depth;) // max depth plumbed during run + VirtualSpace _virtual_space; // Underlying backing store for actual stack bool _overflow; DEBUG_ONLY(bool _drain_in_progress;) @@ -385,6 +386,9 @@ BitMap _region_bm; BitMap _card_bm; + VirtualSpace _virtual_space; // Backing store for worker liveness + // counting structures + // Heap bounds HeapWord* _heap_start; HeapWord* _heap_end; @@ -636,7 +640,7 @@ return _task_queues->steal(task_num, hash_seed, obj); } - ConcurrentMark(ReservedSpace rs, uint max_regions); + ConcurrentMark(ReservedSpace heap_rs, uint max_regions); ~ConcurrentMark(); ConcurrentMarkThread* cmThread() { return _cmThread; }