src/hotspot/share/memory/metaspace.cpp

@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
  * published by the Free Software Foundation.

@@ -141,10 +141,14 @@
   if (new_value < old_capacity_until_GC) {
     // The addition wrapped around, set new_value to aligned max value.
     new_value = align_down(max_uintx, Metaspace::commit_alignment());
   }
 
+  assert(new_value <= MaxMetaspaceSize,
+         "new_value: " SIZE_FORMAT " > MaxMetaspaceSize: " SIZE_FORMAT,
+         new_value, MaxMetaspaceSize);
+
   size_t prev_value = Atomic::cmpxchg(new_value, &_capacity_until_GC, old_capacity_until_GC);
 
   if (old_capacity_until_GC != prev_value) {
     return false;
   }

@@ -234,11 +238,11 @@
   const double minimum_free_percentage = MinMetaspaceFreeRatio / 100.0;
   const double maximum_used_percentage = 1.0 - minimum_free_percentage;
 
   const double min_tmp = used_after_gc / maximum_used_percentage;
   size_t minimum_desired_capacity =
-    (size_t)MIN2(min_tmp, double(max_uintx));
+    (size_t)MIN2(min_tmp, double(MaxMetaspaceSize));
   // Don't shrink less than the initial generation size
   minimum_desired_capacity = MAX2(minimum_desired_capacity,
                                   MetaspaceSize);
 
   log_trace(gc, metaspace)("MetaspaceGC::compute_new_size: ");

@@ -281,11 +285,11 @@
   // Should shrinking be considered?
   if (MaxMetaspaceFreeRatio < 100) {
     const double maximum_free_percentage = MaxMetaspaceFreeRatio / 100.0;
     const double minimum_used_percentage = 1.0 - maximum_free_percentage;
     const double max_tmp = used_after_gc / minimum_used_percentage;
-    size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(max_uintx));
+    size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(MaxMetaspaceSize));
     maximum_desired_capacity = MAX2(maximum_desired_capacity,
                                     MetaspaceSize);
     log_trace(gc, metaspace)("    maximum_free_percentage: %6.2f  minimum_used_percentage: %6.2f",
                              maximum_free_percentage, minimum_used_percentage);
     log_trace(gc, metaspace)("    minimum_desired_capacity: %6.1fKB  maximum_desired_capacity: %6.1fKB",

@@ -1488,10 +1492,13 @@
 
   // Each thread increments the HWM at most once. Even if the thread fails to increment
   // the HWM, an allocation is still attempted. This is because another thread must then
   // have incremented the HWM and therefore the allocation might still succeed.
   do {
+    if (MetaspaceGC::capacity_until_GC() + delta_bytes > MaxMetaspaceSize) {
+      return NULL;
+    }
     incremented = MetaspaceGC::inc_capacity_until_GC(delta_bytes, &after, &before);
     res = allocate(word_size, mdtype);
   } while (!incremented && res == NULL);
 
   if (incremented) {