src/hotspot/share/oops/instanceKlass.cpp

@@ -177,11 +177,12 @@
   // We don't want to resolve any class other than the one being checked.
   for (int i = 0; i < _nest_members->length(); i++) {
     int cp_index = _nest_members->at(i);
     if (_constants->tag_at(cp_index).is_klass()) {
       Klass* k2 = _constants->klass_at(cp_index, THREAD);
-      assert(!HAS_PENDING_EXCEPTION, "Exceptions should not be possible here");
+      assert(!HAS_PENDING_EXCEPTION || PENDING_EXCEPTION->is_a(SystemDictionary::VirtualMachineError_klass()),
+             "Exceptions should not be possible here");
       if (k2 == k) {
         log_trace(class, nestmates)("- class is listed at nest_members[%d] => cp[%d]", i, cp_index);
         return true;
       }
     }

@@ -194,11 +195,12 @@
         // it doesn't match though that should be impossible as it means one classloader
         // has defined two different classes with the same name! A compiler thread won't be
         // able to perform that loading but we can't exclude the compiler threads from
         // executing this logic. But it should actually be impossible to trigger loading here.
         Klass* k2 = _constants->klass_at(cp_index, THREAD);
-        assert(!HAS_PENDING_EXCEPTION, "Exceptions should not be possible here");
+        assert(!HAS_PENDING_EXCEPTION || PENDING_EXCEPTION->is_a(SystemDictionary::VirtualMachineError_klass()),
+               "Exceptions should not be possible here");
         if (k2 == k) {
           log_trace(class, nestmates)("- class is listed as a nest member");
           return true;
         }
         else {

@@ -221,10 +223,11 @@
 // from a more suitable environment later. Otherwise the _nest_host is always
 // set once this method returns.
 // Any errors from nest-host resolution must be preserved so they can be queried
 // from higher-level access checking code, and reported as part of access checking
 // exceptions.
+// VirtualMachineErrors are propagated with a NULL return.
 InstanceKlass* InstanceKlass::nest_host(TRAPS) {
   InstanceKlass* nest_host_k = _nest_host;
   if (nest_host_k != NULL) {
     return nest_host_k;
   }

@@ -253,10 +256,13 @@
                                   _constants->klass_name_at(_nest_host_index)->as_C_string());
     }
 
     Klass* k = _constants->klass_at(_nest_host_index, THREAD);
     if (HAS_PENDING_EXCEPTION) {
+      if (PENDING_EXCEPTION->is_a(SystemDictionary::VirtualMachineError_klass())) {
+        return NULL; // propagate VMEs
+      }
       ResourceMark rm(THREAD);
       stringStream ss;
       char* target_host_class = _constants->klass_name_at(_nest_host_index)->as_C_string();
       ss.print("Nest host resolution of %s with host %s failed: ",
                this->external_name(), target_host_class);

@@ -291,10 +297,13 @@
               return nest_host_k;
             } else {
               error = "current type is not listed as a nest member";
             }
           } else {
+            if (PENDING_EXCEPTION->is_a(SystemDictionary::VirtualMachineError_klass())) {
+              return NULL; // propagate VMEs
+            }
             stringStream ss;
             ss.print("exception on member check: ");
             java_lang_Throwable::print(PENDING_EXCEPTION, &ss);
             error = ss.as_string();
           }

@@ -373,24 +382,25 @@
   _nest_host = host;
 }
 
 // check if 'this' and k are nestmates (same nest_host), or k is our nest_host,
 // or we are k's nest_host - all of which is covered by comparing the two
-// resolved_nest_hosts
+// resolved_nest_hosts.
+// Any exceptions (i.e. VMEs) are propagated.
 bool InstanceKlass::has_nestmate_access_to(InstanceKlass* k, TRAPS) {
 
   assert(this != k, "this should be handled by higher-level code");
 
   // Per JVMS 5.4.4 we first resolve and validate the current class, then
   // the target class k.
 
-  InstanceKlass* cur_host = nest_host(THREAD);
+  InstanceKlass* cur_host = nest_host(CHECK_false);
   if (cur_host == NULL) {
     return false;
   }
 
-  Klass* k_nest_host = k->nest_host(THREAD);
+  Klass* k_nest_host = k->nest_host(CHECK_false);
   if (k_nest_host == NULL) {
     return false;
   }
 
   bool access = (cur_host == k_nest_host);