*** old/src/share/vm/services/memTracker.cpp	Thu Jul 11 08:28:34 2013
--- new/src/share/vm/services/memTracker.cpp	Thu Jul 11 08:28:33 2013

*** 383,392 ****
--- 383,393 ----
   */
  #define MAX_SAFEPOINTS_TO_SKIP     128
  #define SAFE_SEQUENCE_THRESHOLD    30
  #define HIGH_GENERATION_THRESHOLD  60
  #define MAX_RECORDER_THREAD_RATIO  30
+ #define MAX_RECORDER_PER_THREAD    100
  
  void MemTracker::sync() {
    assert(_tracking_level > NMT_off, "NMT is not enabled");
    assert(SafepointSynchronize::is_at_safepoint(), "Safepoint required");
  

*** 435,444 ****
--- 436,450 ----
  
          // see if NMT has too many outstanding recorder instances, it usually
          // means that worker thread is lagging behind in processing them.
          if (!AutoShutdownNMT) {
            _slowdown_calling_thread = (MemRecorder::_instance_count > MAX_RECORDER_THREAD_RATIO * _thread_count);
+         } else {
+           // If auto shutdown is on, enforce MAX_RECORDER_PER_THREAD threshold to prevent OOM
+           if (MemRecorder::_instance_count >= _thread_count * MAX_RECORDER_PER_THREAD) {
+             shutdown(NMT_out_of_memory);
+           } 
          }
  
          // check _worker_thread with lock to avoid racing condition
          if (_worker_thread != NULL) {
            _worker_thread->at_sync_point(pending_recorders, InstanceKlass::number_of_instance_classes());