src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp

rev 2896 : 7121547: G1: High number mispredicted branches while iterating over the marking bitmap
Summary: There is a high number of mispredicted branches associated with calling BitMap::iteratate() from within CMBitMapRO::iterate(). Implement a version of CMBitMapRO::iterate() directly using inline-able routines.
Reviewed-by:

@@ -26,10 +26,39 @@
 #define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP
 
 #include "gc_implementation/g1/concurrentMark.hpp"
 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
 
+inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
+  HeapWord* left  = MAX2(_bmStartWord, mr.start());
+  HeapWord* right = MIN2(_bmStartWord + _bmWordSize, mr.end());
+
+  if (right > left) {
+    // Right-open interval [left-offset, right-offset).
+    BitMap::idx_t start = heapWordToOffset(left);
+    BitMap::idx_t end = heapWordToOffset(right); 
+    
+    start = _bm.get_next_one_offset(start, end);
+    while (start < end) {
+      HeapWord* addr = offsetToHeapWord(start);
+      oop obj = (oop) addr;
+      if (!cl->do_bit(start)) {
+        return false;
+      }
+      HeapWord* next_addr = MIN2(addr + obj->size(), right);
+      BitMap::idx_t next = heapWordToOffset(next_addr);
+      start = _bm.get_next_one_offset(next, end);
+    }
+  }
+  return true;
+}
+
+inline bool CMBitMapRO::iterate(BitMapClosure* cl) {
+  MemRegion mr(startWord(), sizeInWords());
+  return iterate(cl, mr);
+}    
+
 inline void CMTask::push(oop obj) {
   HeapWord* objAddr = (HeapWord*) obj;
   assert(_g1h->is_in_g1_reserved(objAddr), "invariant");
   assert(!_g1h->is_on_master_free_list(
               _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant");