src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp

rev 56898 : 8087198: G1 card refinement: batching, sorting
Reviewed-by: tschatzl

@@ -224,26 +224,109 @@
   _completed_buffers_tail = NULL;
   _num_cards = 0;
   return result;
 }
 
-bool G1DirtyCardQueueSet::refine_buffer(BufferNode* node,
-                                        uint worker_id,
-                                        size_t* total_refined_cards) {
-  G1RemSet* rem_set = G1CollectedHeap::heap()->rem_set();
-  size_t size = buffer_size();
-  void** buffer = BufferNode::make_buffer_from_node(node);
-  size_t i = node->index();
-  assert(i <= size, "invariant");
-  for ( ; (i < size) && !SuspendibleThreadSet::should_yield(); ++i) {
-    CardTable::CardValue* cp = static_cast<CardTable::CardValue*>(buffer[i]);
-    rem_set->refine_card_concurrently(cp, worker_id);
-  }
-  *total_refined_cards += (i - node->index());
-  node->set_index(i);
-  return i == size;
-}
+class G1RefineBufferedCards : public StackObj {
+  BufferNode* const _node;
+  void** const _node_buffer;
+  const size_t _node_buffer_size;
+  size_t* _total_refined_cards;
+  CardTable::CardValue** const _cards;
+  G1RemSet* const _g1rs;
+
+  static inline int compare_card(const void* p1,
+                                 const void* p2) {
+    return *(CardTable::CardValue**)p1 - *(CardTable::CardValue**)p2;
+  }
+
+  void sort_cards(size_t n) {
+    qsort(_cards, n, sizeof(CardTable::CardValue*), compare_card);
+  }
+
+  size_t collect_and_clean_cards() {
+    size_t i = _node->index();
+    size_t num_collected = 0;
+    assert(i <= _node_buffer_size, "invariant");
+    // We don't check for SuspendibleThreadSet::should_yield(), because
+    // collecting, cleaning and abandoning the cards is fast.
+    for ( ; i < _node_buffer_size; ++i) {
+      CardTable::CardValue* cp = static_cast<CardTable::CardValue*>(_node_buffer[i]);
+      if (_g1rs->clean_card_before_refine(cp)) {
+        _cards[num_collected] = cp;
+        num_collected++;
+      } else {
+        // Skipped cards are considered as refined.
+        *_total_refined_cards += 1;
+      }
+    }
+    _node->set_index(i);
+    return num_collected;
+  }
+
+  bool refine_collected_cards(uint worker_id, size_t num_collected) {
+    while (num_collected > 0) {
+      if (SuspendibleThreadSet::should_yield()) {
+        abandon_cards(num_collected);
+        return false;
+      }
+      num_collected--;
+      *_total_refined_cards += 1;
+      _g1rs->refine_card_concurrently(_cards[num_collected], worker_id);
+    }
+    return true;
+  }
+
+  void abandon_cards(size_t num_collected) {
+    assert(num_collected <= _node_buffer_size, "sanity");
+    for (size_t i = 0; i < num_collected; ++i) {
+      *_cards[i] = G1CardTable::dirty_card_val();
+    }
+    size_t buffer_index = _node_buffer_size - num_collected;
+    memcpy(_node_buffer + buffer_index, _cards, num_collected * sizeof(CardTable::CardValue*));
+    _node->set_index(buffer_index);
+  }
+
+public:
+  G1RefineBufferedCards(BufferNode* node,
+                        size_t node_buffer_size,
+                        size_t* total_refined_cards) :
+                        _node(node),
+                        _node_buffer(BufferNode::make_buffer_from_node(node)),
+                        _node_buffer_size(node_buffer_size),
+                        _total_refined_cards(total_refined_cards),
+                        _cards(NEW_RESOURCE_ARRAY(CardTable::CardValue*,
+                                                  node_buffer_size)),
+                        _g1rs(G1CollectedHeap::heap()->rem_set()) {}
+
+  ~G1RefineBufferedCards() {
+    FREE_RESOURCE_ARRAY(CardTable::CardValue*, _cards, _node_buffer_size);
+  }
+
+  // Refine the cards in the BufferNode "_node" from its index to buffer_size.
+  // Stops processing if SuspendibleThreadSet::should_yield() is true.
+  // Returns true if the entire buffer was processed, false if there
+  // is a pending yield request.  The node's index is updated to exclude
+  // the processed elements, e.g. up to the element before processing
+  // stopped, or one past the last element if the entire buffer was
+  // processed. Increments *_total_refined_cards by the number of cards
+  // processed and removed from the buffer.
+  bool refine(uint worker_id) {
+    size_t n = collect_and_clean_cards();
+    // This fence serves two purposes. First, the cards must be cleaned
+    // before processing the contents. Second, we can't proceed with
+    // processing a region until after the read of the region's top in
+    // collect_and_clean_cards(), for synchronization with possibly concurrent
+    // humongous object allocation (see comment at the StoreStore fence before
+    // setting the regions' tops in humongous allocation path).
+    // It's okay that reading region's top and reading region's type were racy
+    // wrto each other. We need both set, in any order, to proceed.
+    OrderAccess::fence();
+    sort_cards(n);
+    return refine_collected_cards(worker_id, n);
+  }
+};
 
 #ifndef ASSERT
 #define assert_fully_consumed(node, buffer_size)
 #else
 #define assert_fully_consumed(node, buffer_size)                \

@@ -276,11 +359,15 @@
 
 bool G1DirtyCardQueueSet::mut_process_buffer(BufferNode* node) {
   uint worker_id = _free_ids.claim_par_id(); // temporarily claim an id
   uint counter_index = worker_id - par_ids_start();
   size_t* counter = &_mutator_refined_cards_counters[counter_index];
-  bool result = refine_buffer(node, worker_id, counter);
+  ResourceMark rm;
+  G1RefineBufferedCards buffered_cards(node,
+                                       buffer_size(),
+                                       counter);
+  bool result = buffered_cards.refine(worker_id);
   _free_ids.release_par_id(worker_id); // release the id
 
   if (result) {
     assert_fully_consumed(node, buffer_size());
   }

@@ -291,20 +378,25 @@
                                                                size_t stop_at,
                                                                size_t* total_refined_cards) {
   BufferNode* node = get_completed_buffer(stop_at);
   if (node == NULL) {
     return false;
-  } else if (refine_buffer(node, worker_id, total_refined_cards)) {
+  } else {
+    G1RefineBufferedCards buffered_cards(node,
+                                         buffer_size(),
+                                         total_refined_cards);
+    if (buffered_cards.refine(worker_id)) {
     assert_fully_consumed(node, buffer_size());
     // Done with fully processed buffer.
     deallocate_buffer(node);
     return true;
   } else {
     // Return partially processed buffer to the queue.
     enqueue_completed_buffer(node);
     return true;
   }
+  }
 }
 
 void G1DirtyCardQueueSet::abandon_logs() {
   assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
   abandon_completed_buffers();