src/hotspot/share/prims/jvm.cpp

@@ -1015,11 +1015,11 @@
   jboolean is_weak = (flags & WEAK_CLASS) == WEAK_CLASS;
   jboolean vm_annotations = (flags & ACCESS_VM_ANNOTATIONS) == ACCESS_VM_ANNOTATIONS;
 
   InstanceKlass* host_class = NULL;
   if (is_nestmate) {
-    host_class = InstanceKlass::cast(lookup_k)->nest_host(THREAD);
+    host_class = InstanceKlass::cast(lookup_k)->nest_host(CHECK_NULL);
   }
 
   if (log_is_enabled(Info, class, nestmates)) {
     log_info(class, nestmates)("LookupDefineClass: %s - %s%s, %s, %s, %s",
                                name,

@@ -2050,12 +2050,12 @@
   JVMWrapper("JVM_GetNestHost");
   Klass* c = java_lang_Class::as_Klass(JNIHandles::resolve_non_null(current));
   assert(c->is_instance_klass(), "must be");
   InstanceKlass* ck = InstanceKlass::cast(c);
   InstanceKlass* host = ck->nest_host(THREAD);
-  assert(host != NULL, "host should never be NULL");
-  return (jclass) JNIHandles::make_local(THREAD, host->java_mirror());
+  return (jclass) (host == NULL ? NULL :
+                   JNIHandles::make_local(THREAD, host->java_mirror()));
 }
 JVM_END
 
 JVM_ENTRY(jobjectArray, JVM_GetNestMembers(JNIEnv* env, jclass current))
 {

@@ -2092,10 +2092,13 @@
       int count = 0;
       for (int i = 0; i < length; i++) {
         int cp_index = members->at(i);
         Klass* k = host->constants()->klass_at(cp_index, THREAD);
         if (HAS_PENDING_EXCEPTION) {
+          if (PENDING_EXCEPTION->is_a(SystemDictionary::VirtualMachineError_klass())) {
+            return NULL; // propagate VMEs
+          }
           if (doLog) {
             ResourceMark rm(THREAD);
             stringStream ss;
             char* target_member_class = host->constants()->klass_name_at(cp_index)->as_C_string();
             ss.print(" - resolution of nest member %s failed: ", target_member_class);

@@ -2105,11 +2108,11 @@
           CLEAR_PENDING_EXCEPTION;
           continue;
         }
         if (k->is_instance_klass()) {
           InstanceKlass* ik = InstanceKlass::cast(k);
-          InstanceKlass* nest_host_k = ik->nest_host(THREAD);
+          InstanceKlass* nest_host_k = ik->nest_host(CHECK_NULL);
           if (nest_host_k == host) {
             result->obj_at_put(count+1, k->java_mirror());
             count++;
             if (doLog) {
               ResourceMark rm(THREAD);