src/hotspot/share/utilities/debug.cpp

rev 55480 : [mq]: 8227275-native-oom-hanging-assertions

@@ -732,10 +732,14 @@
       g_assert_poison = page;
     }
   }
 }
 
+void disarm_assert_poison() {
+  g_assert_poison = &g_dummy;
+}
+
 static void store_context(const void* context) {
   memcpy(&g_stored_assertion_context, context, sizeof(ucontext_t));
 #if defined(__linux) && defined(PPC64)
   // on Linux ppc64, ucontext_t contains pointers into itself which have to be patched up
   //  after copying the context (see comment in sys/ucontext.h):

@@ -744,11 +748,18 @@
 }
 
 bool handle_assert_poison_fault(const void* ucVoid, const void* faulting_address) {
   if (faulting_address == g_assert_poison) {
     // Disarm poison page.
-    os::protect_memory((char*)g_assert_poison, os::vm_page_size(), os::MEM_PROT_RWX);
+    if (os::protect_memory((char*)g_assert_poison, os::vm_page_size(), os::MEM_PROT_RWX) == false) {
+#ifdef ASSERT
+      fprintf(stderr, "Assertion poison page cannot be unprotected - mprotect failed with %d (%s)",
+              errno, os::strerror(errno));
+      fflush(stderr);
+#endif
+      return false; // unprotecting memory may fail in OOM situations, as surprising as this sounds.
+    }
     // Store Context away.
     if (ucVoid) {
       const intx my_tid = os::current_thread_id();
       if (Atomic::cmpxchg(my_tid, &g_asserting_thread, (intx)0) == 0) {
         store_context(ucVoid);