src/hotspot/share/memory/metaspace.cpp

@@ -1275,19 +1275,24 @@
 
   if (result == NULL) {
     tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype);
 
     // Allocation failed.
-    if (is_init_completed() && !(DumpSharedSpaces && THREAD->is_VM_thread())) {
+    if (is_init_completed()) {
       // Only start a GC if the bootstrapping has completed.
-      // Also, we cannot GC if we are at the end of the CDS dumping stage which runs inside
-      // the VM thread.
 
-      // Try to clean out some memory and retry.
+      if (DumpSharedSpaces && THREAD->is_VM_thread()) {
+        // We cannot GC if we are at the end of the CDS dumping stage which runs inside
+        // the VM thread, so just expand. We won't be allocating much anyway.
+        result = loader_data->metaspace_non_null()->expand_and_allocate(word_size, mdtype);
+      } else {
+        // Try to clean out some heap memory and retry. This can prevent premature
+        // expansion of the metaspace.
       result = Universe::heap()->satisfy_failed_metadata_allocation(loader_data, word_size, mdtype);
     }
   }
+  }
 
   if (result == NULL) {
     if (DumpSharedSpaces) {
       // CDS dumping keeps loading classes, so if we hit an OOM we probably will keep hitting OOM.
       // We should abort to avoid generating a potentially bad archive.