# HG changeset patch
# User zgu
# Date 1590504883 14400
#      Tue May 26 10:54:43 2020 -0400
# Node ID 43c773f8ce73bdedaf72b2f46b038cabf08cff80
# Parent  74984c90b4825c48dac92859682d72469732c3bf
8245823: Shenandoah: inline/optimize ShenandoahEvacOOMScope

diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
@@ -134,7 +134,7 @@
         _heap->in_collection_set(obj) &&
         obj == fwd) {
       Thread *t = Thread::current();
-      ShenandoahEvacOOMScope oom_evac_scope;
+      ShenandoahEvacOOMScope oom_evac_scope(_heap, t);
       return _heap->evacuate_object(obj, t);
     } else {
       return fwd;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
@@ -29,6 +29,7 @@
 #include "gc/shenandoah/shenandoahAsserts.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
@@ -62,8 +63,9 @@
   if (obj == fwd) {
     assert(_heap->is_evacuation_in_progress(),
            "evac should be in progress");
-    ShenandoahEvacOOMScope scope;
-    fwd = _heap->evacuate_object(obj, Thread::current());
+    Thread* const t = Thread::current();
+    ShenandoahEvacOOMScope scope(_heap, t);
+    fwd = _heap->evacuate_object(obj, t);
   }
 
   if (load_addr != NULL && fwd != obj) {
@@ -337,7 +339,7 @@
 void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) {
   assert(_heap->is_evacuation_in_progress(), "only during evacuation");
   if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) {
-    ShenandoahEvacOOMScope oom_evac;
+    ShenandoahEvacOOMScope oom_evac(_heap);
     arraycopy_work<T, true, true, false>(src, count);
   }
 }
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,7 @@
 
 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "memory/iterator.hpp"
 #include "oops/access.hpp"
@@ -84,7 +84,7 @@
 void ShenandoahBarrierSet::clone_evacuation(oop obj) {
   assert(_heap->is_evacuation_in_progress(), "only during evacuation");
   if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
-    ShenandoahEvacOOMScope oom_evac_scope;
+    ShenandoahEvacOOMScope oom_evac_scope(_heap);
     ShenandoahUpdateRefsForOopClosure</* has_fwd = */ true, /* evac = */ true, /* enqueue */ false> cl;
     obj->oop_iterate(&cl);
   }
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
@@ -28,7 +28,7 @@
 #include "code/nmethod.hpp"
 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,9 +26,7 @@
 
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
-#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
-#include "runtime/atomic.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "runtime/os.hpp"
 #include "runtime/thread.hpp"
 
@@ -48,21 +46,9 @@
   ShenandoahThreadLocalData::set_oom_during_evac(Thread::current(), true);
 }
 
-void ShenandoahEvacOOMHandler::enter_evacuation() {
+void ShenandoahEvacOOMHandler::register_thread_to_protocol(Thread* thr) {
   jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
 
-  Thread* const thr = Thread::current();
-  uint8_t level = ShenandoahThreadLocalData::push_evac_oom_scope(thr);
-  if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
-    wait_for_no_evac_threads();
-    return;
-  }
-
-  // Nesting case, this thread already registered
-  if (level > 0) {
-     return;
-  }
-
   assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must not be set");
   while (true) {
     jint other = Atomic::cmpxchg(&_threads_in_evac, threads_in_evac, threads_in_evac + 1);
@@ -82,14 +68,7 @@
   }
 }
 
-void ShenandoahEvacOOMHandler::leave_evacuation() {
-  Thread* const thr = Thread::current();
-  uint8_t level = ShenandoahThreadLocalData::pop_evac_oom_scope(thr);
-  // Not top level, just return
-  if (level > 1) {
-    return;
-  }
-
+void ShenandoahEvacOOMHandler::unregister_thread_to_protocol(Thread* thr) {
   if (!ShenandoahThreadLocalData::is_oom_during_evac(thr)) {
     assert((Atomic::load_acquire(&_threads_in_evac) & ~OOM_MARKER_MASK) > 0, "sanity");
     // NOTE: It's ok to simply decrement, even with mask set, because unmasked value is positive.
@@ -126,11 +105,3 @@
   assert((Atomic::load_acquire(&_threads_in_evac) & ~OOM_MARKER_MASK) == 0, "sanity");
   Atomic::release_store_fence(&_threads_in_evac, (jint)0);
 }
-
-ShenandoahEvacOOMScope::ShenandoahEvacOOMScope() {
-  ShenandoahHeap::heap()->enter_evacuation();
-}
-
-ShenandoahEvacOOMScope::~ShenandoahEvacOOMScope() {
-  ShenandoahHeap::heap()->leave_evacuation();
-}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
 
 #include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
+#include "runtime/thread.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 /**
@@ -98,12 +99,12 @@
    * When this method returns false, evacuation must not be entered, and caller
    * may safely continue with a simple resolve (if Java thread).
    */
-  void enter_evacuation();
+  inline void enter_evacuation(Thread* t);
 
   /**
    * Leave evacuation path.
    */
-  void leave_evacuation();
+  inline void leave_evacuation(Thread* t);
 
   /**
    * Signal out-of-memory during evacuation. It will prevent any other threads
@@ -113,12 +114,25 @@
   void handle_out_of_memory_during_evacuation();
 
   void clear();
+
+private:
+  // Register/Unregister thread to evacuation OOM protocol
+  void register_thread_to_protocol(Thread* t);
+  void unregister_thread_to_protocol(Thread* t);
 };
 
+class ShenandoahHeap;
+
 class ShenandoahEvacOOMScope : public StackObj {
+private:
+  ShenandoahHeap* const _heap;
+  Thread* const         _thread;
 public:
-  ShenandoahEvacOOMScope();
-  ~ShenandoahEvacOOMScope();
+  inline ShenandoahEvacOOMScope();
+  inline ShenandoahEvacOOMScope(Thread* t);
+  inline ShenandoahEvacOOMScope(ShenandoahHeap* heap);
+  inline ShenandoahEvacOOMScope(ShenandoahHeap* heap, Thread* t);
+  inline ~ShenandoahEvacOOMScope();
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp
new file mode 100644
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2020, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP
+
+#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "runtime/atomic.hpp"
+
+void ShenandoahEvacOOMHandler::enter_evacuation(Thread* thr) {
+  jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
+
+  uint8_t level = ShenandoahThreadLocalData::push_evac_oom_scope(thr);
+  if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
+    wait_for_no_evac_threads();
+    return;
+  }
+
+  // Nesting case, this thread already registered
+  if (level != 0) {
+     return;
+  }
+  // Entering top level scope, register this thread.
+  register_thread_to_protocol(thr);
+}
+
+void ShenandoahEvacOOMHandler::leave_evacuation(Thread* thr) {
+  uint8_t level = ShenandoahThreadLocalData::pop_evac_oom_scope(thr);
+  // Not top level, just return
+  if (level > 1) {
+    return;
+  }
+
+  // Leaving top level scope, unregister this thread.
+  unregister_thread_to_protocol(thr);
+}
+
+ShenandoahEvacOOMScope::ShenandoahEvacOOMScope() :
+  _heap(ShenandoahHeap::heap()),
+  _thread(Thread::current()) {
+  _heap->enter_evacuation(_thread);
+}
+
+ShenandoahEvacOOMScope::ShenandoahEvacOOMScope(Thread* t) :
+  _heap(ShenandoahHeap::heap()),
+  _thread(t) {
+  _heap->enter_evacuation(_thread);
+}
+
+ShenandoahEvacOOMScope::ShenandoahEvacOOMScope(ShenandoahHeap* heap) :
+  _heap(heap),
+  _thread(Thread::current()) {
+  _heap->enter_evacuation(_thread);
+}
+
+ShenandoahEvacOOMScope::ShenandoahEvacOOMScope(ShenandoahHeap* heap, Thread* t) :
+  _heap(heap),
+  _thread(t) {
+  _heap->enter_evacuation(_thread);
+}
+
+ShenandoahEvacOOMScope::~ShenandoahEvacOOMScope() {
+  _heap->leave_evacuation(_thread);
+}
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -970,11 +970,11 @@
     if (_concurrent) {
       ShenandoahConcurrentWorkerSession worker_session(worker_id);
       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
-      ShenandoahEvacOOMScope oom_evac_scope;
+      ShenandoahEvacOOMScope oom_evac_scope(_sh);
       do_work();
     } else {
       ShenandoahParallelWorkerSession worker_session(worker_id);
-      ShenandoahEvacOOMScope oom_evac_scope;
+      ShenandoahEvacOOMScope oom_evac_scope(_sh);
       do_work();
     }
   }
@@ -3020,14 +3020,6 @@
   return _memory_pool->get_memory_usage();
 }
 
-void ShenandoahHeap::enter_evacuation() {
-  _oom_evac_handler.enter_evacuation();
-}
-
-void ShenandoahHeap::leave_evacuation() {
-  _oom_evac_handler.leave_evacuation();
-}
-
 ShenandoahRegionIterator::ShenandoahRegionIterator() :
   _heap(ShenandoahHeap::heap()),
   _index(0) {}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
@@ -699,8 +699,8 @@
   inline oop evacuate_object(oop src, Thread* thread);
 
   // Call before/after evacuation.
-  void enter_evacuation();
-  void leave_evacuation();
+  inline void enter_evacuation(Thread* t);
+  inline void leave_evacuation(Thread* t);
 
 // ---------- Helper functions
 //
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
@@ -85,6 +85,14 @@
   return result;
 }
 
+inline void ShenandoahHeap::enter_evacuation(Thread* t) {
+  _oom_evac_handler.enter_evacuation(t);
+}
+
+inline void ShenandoahHeap::leave_evacuation(Thread* t) {
+  _oom_evac_handler.leave_evacuation(t);
+}
+
 template <class T>
 inline oop ShenandoahHeap::update_with_forwarded_not_null(T* p, oop obj) {
   if (in_collection_set(obj)) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
@@ -196,7 +196,7 @@
       data->oops_do(&cl);
     }
   } else if (heap->is_concurrent_weak_root_in_progress()) {
-    ShenandoahEvacOOMScope evac_scope;
+    ShenandoahEvacOOMScope evac_scope(heap);
     heal_nmethod_metadata(data);
   } else {
     // There is possibility that GC is cancelled when it arrives final mark.