diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp
index 2089c0df864..2eb748dbdce 100644
--- a/src/hotspot/share/runtime/arguments.cpp
+++ b/src/hotspot/share/runtime/arguments.cpp
@@ -1054,13 +1054,15 @@ void Arguments::add_string(char*** bldarray, int* count, const char* arg) {
     return;
   }
 
-  int new_count = *count + 1;
+  int old_count = *count;
+  int new_count = old_count + 1;
 
   // expand the array and add arg to the last element
-  if (*bldarray == NULL) {
-    *bldarray = NEW_C_HEAP_ARRAY(char*, new_count, mtArguments);
-  } else {
-    *bldarray = REALLOC_C_HEAP_ARRAY(char*, *bldarray, new_count, mtArguments);
+  //  (expand in pow 2 steps to save on realloc calls)
+  size_t arraySizeNow = next_power_of_2(old_count);
+  size_t arraySizeNeeded = next_power_of_2(new_count);
+  if (arraySizeNow < arraySizeNeeded) {
+    *bldarray = REALLOC_C_HEAP_ARRAY(char*, *bldarray, arraySizeNeeded, mtArguments);
   }
   (*bldarray)[*count] = os::strdup_check_oom(arg);
   *count = new_count;
@@ -1981,17 +1983,6 @@ bool Arguments::check_vm_args_consistency() {
     status = false;
   }
 
-  if (PrintNMTStatistics) {
-#if INCLUDE_NMT
-    if (MemTracker::tracking_level() == NMT_off) {
-#endif // INCLUDE_NMT
-      warning("PrintNMTStatistics is disabled, because native memory tracking is not enabled");
-      PrintNMTStatistics = false;
-#if INCLUDE_NMT
-    }
-#endif
-  }
-
   status = CompilerConfig::check_args_consistency(status);
 #if INCLUDE_JVMCI
   if (status && EnableJVMCI) {
@@ -3712,29 +3703,6 @@ jint Arguments::match_special_option_and_act(const JavaVMInitArgs* args,
       JVMFlag::printFlags(tty, false);
       vm_exit(0);
     }
-    if (match_option(option, "-XX:NativeMemoryTracking", &tail)) {
-#if INCLUDE_NMT
-      // The launcher did not setup nmt environment variable properly.
-      if (!MemTracker::check_launcher_nmt_support(tail)) {
-        warning("Native Memory Tracking did not setup properly, using wrong launcher?");
-      }
-
-      // Verify if nmt option is valid.
-      if (MemTracker::verify_nmt_option()) {
-        // Late initialization, still in single-threaded mode.
-        if (MemTracker::tracking_level() >= NMT_summary) {
-          MemTracker::init();
-        }
-      } else {
-        vm_exit_during_initialization("Syntax error, expecting -XX:NativeMemoryTracking=[off|summary|detail]", NULL);
-      }
-      continue;
-#else
-      jio_fprintf(defaultStream::error_stream(),
-        "Native Memory Tracking is not supported in this VM\n");
-      return JNI_ERR;
-#endif
-    }
 
 #ifndef PRODUCT
     if (match_option(option, "-XX:+PrintFlagsWithComments")) {
diff --git a/src/hotspot/share/runtime/os.cpp b/src/hotspot/share/runtime/os.cpp
index d90780215b4..192958a387f 100644
--- a/src/hotspot/share/runtime/os.cpp
+++ b/src/hotspot/share/runtime/os.cpp
@@ -65,6 +65,7 @@
 #include "services/mallocTracker.hpp"
 #include "services/memTracker.hpp"
 #include "services/nmtCommon.hpp"
+#include "services/nmtPreInitBuffer.hpp"
 #include "services/threadService.hpp"
 #include "utilities/align.hpp"
 #include "utilities/count_trailing_zeros.hpp"
@@ -679,17 +680,21 @@ void* os::malloc(size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
   NOT_PRODUCT(inc_stat_counter(&num_mallocs, 1));
   NOT_PRODUCT(inc_stat_counter(&alloc_bytes, size));
 
+  // Handle malloc(0) the same on all platforms
+  size = MAX2(size, (size_t)1);
+
+#if INCLUDE_NMT
+  // Before NMT initialization, allocate solely from the NMT pre-init buffer.
+  if (!MemTracker::is_initialized()) {
+    return NMTPreInitBuffer::allocate(size, memflags);
+  }
+#endif
+
   // Since os::malloc can be called when the libjvm.{dll,so} is
   // first loaded and we don't have a thread yet we must accept NULL also here.
   assert(!os::ThreadCrashProtection::is_crash_protected(Thread::current_or_null()),
          "malloc() not allowed when crash protection is set");
 
-  if (size == 0) {
-    // return a valid pointer if size is zero
-    // if NULL is returned the calling functions assume out of memory.
-    size = 1;
-  }
-
   // NMT support
   NMT_TrackingLevel level = MemTracker::tracking_level();
   size_t            nmt_header_size = MemTracker::malloc_header_size(level);
@@ -738,6 +743,21 @@ void* os::realloc(void *memblock, size_t size, MEMFLAGS flags) {
 
 void* os::realloc(void *memblock, size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
 
+  if (memblock == NULL) {
+    return os::malloc(size, memflags, stack);
+  }
+
+#if INCLUDE_NMT
+  // Before NMT initialization, allocate solely from the NMT pre-init buffer.
+  if (!MemTracker::is_initialized()) {
+    return NMTPreInitBuffer::reallocate(memblock, size, memflags);
+  }
+  // After NMT initialization, evacuate blocks to C-Heap on realloc
+  if (NMTPreInitBuffer::contains(memblock)) {
+    return NMTPreInitBuffer::reallocate_to_c_heap(memblock, size, memflags);
+  }
+#endif
+
   // For the test flag -XX:MallocMaxTestWords
   if (has_reached_max_malloc_test_peak(size)) {
     return NULL;
@@ -759,9 +779,6 @@ void* os::realloc(void *memblock, size_t size, MEMFLAGS memflags, const NativeCa
   void* ptr = ::realloc(membase, size + nmt_header_size);
   return MemTracker::record_malloc(ptr, size, memflags, stack, level);
 #else
-  if (memblock == NULL) {
-    return os::malloc(size, memflags, stack);
-  }
   if ((intptr_t)memblock == (intptr_t)MallocCatchPtr) {
     log_warning(malloc, free)("os::realloc caught " PTR_FORMAT, p2i(memblock));
     breakpoint();
@@ -788,6 +805,20 @@ void* os::realloc(void *memblock, size_t size, MEMFLAGS memflags, const NativeCa
 
 // handles NULL pointers
 void  os::free(void *memblock) {
+
+  // free(NULL) is a noop
+  if (!memblock) {
+    return;
+  }
+
+#if INCLUDE_NMT
+  // Freeing blocks from the NMT pre-init buffer should be done by the buffer.
+  if (NMTPreInitBuffer::contains(memblock)) {
+    NMTPreInitBuffer::free(memblock);
+    return;
+  }
+#endif
+
   NOT_PRODUCT(inc_stat_counter(&num_frees, 1));
 #ifdef ASSERT
   if (memblock == NULL) return;
diff --git a/src/hotspot/share/runtime/thread.cpp b/src/hotspot/share/runtime/thread.cpp
index 6ed3cd11386..06190126fc5 100644
--- a/src/hotspot/share/runtime/thread.cpp
+++ b/src/hotspot/share/runtime/thread.cpp
@@ -2758,6 +2758,31 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   jint parse_result = Arguments::parse(args);
   if (parse_result != JNI_OK) return parse_result;
 
+#if INCLUDE_NMT
+  // Initialize NMT
+  if (MemTracker::is_initialized()) {
+    // The pre-init NMT buffer had been exhausted before NMT arguments
+    // could be parsed; in that case NMT is switched off.
+    assert(MemTracker::tracking_level() == NMT_off, "sanity");
+    if (!FLAG_IS_DEFAULT(NativeMemoryTracking)) {
+      warning("-XX:NativeMemoryTracking ignored due to early buffer exhaustion."
+              " Native Memory Tracking disabled.");
+    }
+  } else {
+    const NMT_TrackingLevel lvl = MemTracker::parse_level_string(NativeMemoryTracking);
+    if (lvl == NMT_unknown) {
+      vm_exit_during_initialization("Syntax error, expecting -XX:NativeMemoryTracking=[off|summary|detail]", NULL);
+    }
+    if (!MemTracker::initialize(lvl)) {
+      vm_exit_during_initialization("NMT initialization failed", NULL);
+    }
+  }
+#else
+  if (!FLAG_IS_DEFAULT(NativeMemoryTracking)) {
+    warning("Native Memory Tracking is not supported in this VM");
+  }
+#endif // INCLUDE_NMT
+
   os::init_before_ergo();
 
   jint ergo_result = Arguments::apply_ergo();
diff --git a/src/hotspot/share/services/mallocSiteTable.cpp b/src/hotspot/share/services/mallocSiteTable.cpp
index 869a6738531..41d2ee96fe1 100644
--- a/src/hotspot/share/services/mallocSiteTable.cpp
+++ b/src/hotspot/share/services/mallocSiteTable.cpp
@@ -29,7 +29,7 @@
 #include "services/mallocSiteTable.hpp"
 
 // Malloc site hashtable buckets
-MallocSiteHashtableEntry*  MallocSiteTable::_table[MallocSiteTable::table_size];
+MallocSiteHashtableEntry** MallocSiteTable::_table = NULL;
 const NativeCallStack* MallocSiteTable::_hash_entry_allocation_stack = NULL;
 const MallocSiteHashtableEntry* MallocSiteTable::_hash_entry_allocation_site = NULL;
 
@@ -51,6 +51,13 @@ NOT_PRODUCT(int MallocSiteTable::_peak_count = 0;)
 bool MallocSiteTable::initialize() {
   assert((size_t)table_size <= MAX_MALLOCSITE_TABLE_SIZE, "Hashtable overflow");
 
+  // Allocate space for the hash table. We use raw malloc here to prevent
+  // circularities.
+  _table = (MallocSiteHashtableEntry**)::calloc(table_size, sizeof(MallocSiteHashtableEntry));
+  if (_table == NULL) {
+    return false;
+  }
+
   // Fake the call stack for hashtable entry allocation
   assert(NMT_TrackingStackDepth > 1, "At least one tracking stack");
 
diff --git a/src/hotspot/share/services/mallocSiteTable.hpp b/src/hotspot/share/services/mallocSiteTable.hpp
index f401d456b18..fcc672f27e8 100644
--- a/src/hotspot/share/services/mallocSiteTable.hpp
+++ b/src/hotspot/share/services/mallocSiteTable.hpp
@@ -253,7 +253,7 @@ class MallocSiteTable : AllStatic {
 
   // The callsite hashtable. It has to be a static table,
   // since malloc call can come from C runtime linker.
-  static MallocSiteHashtableEntry*        _table[table_size];
+  static MallocSiteHashtableEntry**       _table;
   static const NativeCallStack*           _hash_entry_allocation_stack;
   static const MallocSiteHashtableEntry*  _hash_entry_allocation_site;
 
diff --git a/src/hotspot/share/services/memTracker.cpp b/src/hotspot/share/services/memTracker.cpp
index 981584f94be..f5af747c0f3 100644
--- a/src/hotspot/share/services/memTracker.cpp
+++ b/src/hotspot/share/services/memTracker.cpp
@@ -32,6 +32,7 @@
 #include "services/memReporter.hpp"
 #include "services/mallocTracker.inline.hpp"
 #include "services/memTracker.hpp"
+#include "services/nmtPreInitBuffer.hpp"
 #include "services/threadStackTracker.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/defaultStream.hpp"
@@ -45,81 +46,46 @@ volatile NMT_TrackingLevel MemTracker::_tracking_level = NMT_unknown;
 NMT_TrackingLevel MemTracker::_cmdline_tracking_level = NMT_unknown;
 
 MemBaseline MemTracker::_baseline;
-bool MemTracker::_is_nmt_env_valid = true;
 
-static const size_t buffer_size = 64;
+// Returns the parsed level; NMT_unknown if string is invalid
+NMT_TrackingLevel MemTracker::parse_level_string(const char* s) {
+  if (strcmp(s, "summary") == 0) {
+    return NMT_summary;
+  } else if (strcmp(s, "detail") == 0) {
+    return NMT_detail;
+  } else if (strcmp(s, "off") == 0) {
+    return NMT_off;
+  }
+  return NMT_unknown;
+}
+
+bool MemTracker::initialize(NMT_TrackingLevel level) {
+  assert(_tracking_level == NMT_unknown, "only call once");
+  assert(level == NMT_off || level == NMT_summary || level == NMT_detail, "sanity");
 
-NMT_TrackingLevel MemTracker::init_tracking_level() {
   // Memory type is encoded into tracking header as a byte field,
   // make sure that we don't overflow it.
   STATIC_ASSERT(mt_number_of_types <= max_jubyte);
 
-  char nmt_env_variable[buffer_size];
-  jio_snprintf(nmt_env_variable, sizeof(nmt_env_variable), "NMT_LEVEL_%d", os::current_process_id());
-  const char* nmt_env_value;
-#ifdef _WINDOWS
-  // Read the NMT environment variable from the PEB instead of the CRT
-  char value[buffer_size];
-  nmt_env_value = GetEnvironmentVariable(nmt_env_variable, value, (DWORD)sizeof(value)) != 0 ? value : NULL;
-#else
-  nmt_env_value = ::getenv(nmt_env_variable);
-#endif
-  NMT_TrackingLevel level = NMT_off;
-  if (nmt_env_value != NULL) {
-    if (strcmp(nmt_env_value, "summary") == 0) {
-      level = NMT_summary;
-    } else if (strcmp(nmt_env_value, "detail") == 0) {
-      level = NMT_detail;
-    } else if (strcmp(nmt_env_value, "off") != 0) {
-      // The value of the environment variable is invalid
-      _is_nmt_env_valid = false;
+  if (level >= NMT_off) {
+    if (!MallocTracker::initialize(level) ||
+        !VirtualMemoryTracker::initialize(level)) {
+      _tracking_level = NMT_off;
     }
-    // Remove the environment variable to avoid leaking to child processes
-    os::unsetenv(nmt_env_variable);
-  }
-
-  if (!MallocTracker::initialize(level) ||
-      !VirtualMemoryTracker::initialize(level)) {
-    level = NMT_off;
-  }
-  return level;
-}
-
-void MemTracker::init() {
-  NMT_TrackingLevel level = tracking_level();
-  if (level >= NMT_summary) {
-    if (!VirtualMemoryTracker::late_initialize(level) ||
-        !ThreadStackTracker::late_initialize(level)) {
-      shutdown();
-      return;
+    if (level >= NMT_summary) {
+      if (!VirtualMemoryTracker::late_initialize(level) ||
+          !ThreadStackTracker::late_initialize(level)) {
+        shutdown();
+        return false;
+      }
     }
   }
-}
 
-bool MemTracker::check_launcher_nmt_support(const char* value) {
-  if (strcmp(value, "=detail") == 0) {
-    if (MemTracker::tracking_level() != NMT_detail) {
-      return false;
-    }
-  } else if (strcmp(value, "=summary") == 0) {
-    if (MemTracker::tracking_level() != NMT_summary) {
-      return false;
-    }
-  } else if (strcmp(value, "=off") == 0) {
-    if (MemTracker::tracking_level() != NMT_off) {
-      return false;
-    }
-  } else {
-    _is_nmt_env_valid = false;
-  }
+  _tracking_level = _cmdline_tracking_level = level;
 
   return true;
 }
 
-bool MemTracker::verify_nmt_option() {
-  return _is_nmt_env_valid;
-}
-
 void* MemTracker::malloc_base(void* memblock) {
   return MallocTracker::get_base(memblock);
 }
@@ -138,7 +104,6 @@ void Tracker::record(address addr, size_t size) {
   }
 }
 
-
 // Shutdown can only be issued via JCmd, and NMT JCmd is serialized by lock
 void MemTracker::shutdown() {
   // We can only shutdown NMT to minimal tracking level if it is ever on.
@@ -219,4 +184,7 @@ void MemTracker::tuning_statistics(outputStream* out) {
   NOT_PRODUCT(out->print_cr("Peak concurrent access: %d", MallocSiteTable::access_peak_count());)
   out->cr();
   MallocSiteTable::print_tuning_statistics(out);
+  out->cr();
+  NMTPreInitBuffer::print_state(out);
+  out->cr();
 }
diff --git a/src/hotspot/share/services/memTracker.hpp b/src/hotspot/share/services/memTracker.hpp
index b15bc1dcc26..db46ac4fa2a 100644
--- a/src/hotspot/share/services/memTracker.hpp
+++ b/src/hotspot/share/services/memTracker.hpp
@@ -115,42 +115,37 @@ class Tracker : public StackObj {
 class MemTracker : AllStatic {
   friend class VirtualMemoryTrackerTest;
 
+  static void assert_post_init() {
+    assert(is_initialized(), "NMT not yet initialized.");
+  }
+
  public:
+
+  // Initialize NMT to a given tracking level. Only call once.
+  // Allowed transitions here are:
+  //  unknown -> off
+  //  unknown -> summary
+  //  unknown -> detail
+  static bool initialize(NMT_TrackingLevel level);
+
+  // Returns true if NMT had been initialized.
+  static bool is_initialized()  {
+    return _tracking_level != NMT_unknown;
+  }
+
+  // Returns the parsed level; NMT_unknown if string is invalid
+  static NMT_TrackingLevel parse_level_string(const char* s);
+
   static inline NMT_TrackingLevel tracking_level() {
-    if (_tracking_level == NMT_unknown) {
-      // No fencing is needed here, since JVM is in single-threaded
-      // mode.
-      _tracking_level = init_tracking_level();
-      _cmdline_tracking_level = _tracking_level;
-    }
     return _tracking_level;
   }
 
-  // A late initialization, for the stuff(s) can not be
-  // done in init_tracking_level(), which can NOT malloc
-  // any memory.
-  static void init();
-
-  // Shutdown native memory tracking
+  // Shutdown native memory tracking.
+  // This transitions the tracking level:
+  //  summary -> minimal
+  //  detail  -> minimal
   static void shutdown();
 
-  // Verify native memory tracking command line option.
-  // This check allows JVM to detect if compatible launcher
-  // is used.
-  // If an incompatible launcher is used, NMT may not be
-  // able to start, even it is enabled by command line option.
-  // A warning message should be given if it is encountered.
-  static bool check_launcher_nmt_support(const char* value);
-
-  // This method checks native memory tracking environment
-  // variable value passed by launcher.
-  // Launcher only obligated to pass native memory tracking
-  // option value, but not obligated to validate the value,
-  // and launcher has option to discard native memory tracking
-  // option from the command line once it sets up the environment
-  // variable, so NMT has to catch the bad value here.
-  static bool verify_nmt_option();
-
   // Transition the tracking level to specified level
   static bool transition_to(NMT_TrackingLevel level);
 
@@ -207,8 +202,11 @@ class MemTracker : AllStatic {
     MallocTracker::record_arena_size_change(diff, flag);
   }
 
+  // Note: we expect no virtual memory operations to happen before NMT initialization.
+
   static inline void record_virtual_memory_reserve(void* addr, size_t size, const NativeCallStack& stack,
     MEMFLAGS flag = mtNone) {
+    assert_post_init();
     if (tracking_level() < NMT_summary) return;
     if (addr != NULL) {
       ThreadCritical tc;
@@ -220,6 +218,7 @@ class MemTracker : AllStatic {
 
   static inline void record_virtual_memory_reserve_and_commit(void* addr, size_t size,
     const NativeCallStack& stack, MEMFLAGS flag = mtNone) {
+    assert_post_init();
     if (tracking_level() < NMT_summary) return;
     if (addr != NULL) {
       ThreadCritical tc;
@@ -231,6 +230,7 @@ class MemTracker : AllStatic {
 
   static inline void record_virtual_memory_commit(void* addr, size_t size,
     const NativeCallStack& stack) {
+    assert_post_init();
     if (tracking_level() < NMT_summary) return;
     if (addr != NULL) {
       ThreadCritical tc;
@@ -246,6 +246,7 @@ class MemTracker : AllStatic {
   // The two new memory regions will be both registered under stack and
   //  memory flags of the original region.
   static inline void record_virtual_memory_split_reserved(void* addr, size_t size, size_t split) {
+    assert_post_init();
     if (tracking_level() < NMT_summary) return;
     if (addr != NULL) {
       ThreadCritical tc;
@@ -256,6 +257,7 @@ class MemTracker : AllStatic {
   }
 
   static inline void record_virtual_memory_type(void* addr, MEMFLAGS flag) {
+    assert_post_init();
     if (tracking_level() < NMT_summary) return;
     if (addr != NULL) {
       ThreadCritical tc;
@@ -265,6 +267,7 @@ class MemTracker : AllStatic {
   }
 
   static void record_thread_stack(void* addr, size_t size) {
+    assert_post_init();
     if (tracking_level() < NMT_summary) return;
     if (addr != NULL) {
       ThreadStackTracker::new_thread_stack((address)addr, size, CALLER_PC);
@@ -272,6 +275,7 @@ class MemTracker : AllStatic {
   }
 
   static inline void release_thread_stack(void* addr, size_t size) {
+    assert_post_init();
     if (tracking_level() < NMT_summary) return;
     if (addr != NULL) {
       ThreadStackTracker::delete_thread_stack((address)addr, size);
diff --git a/src/hotspot/share/services/nmtCommon.hpp b/src/hotspot/share/services/nmtCommon.hpp
index 7d4f02f1217..f5b5abef966 100644
--- a/src/hotspot/share/services/nmtCommon.hpp
+++ b/src/hotspot/share/services/nmtCommon.hpp
@@ -32,12 +32,48 @@
 #define CALC_OBJ_SIZE_IN_TYPE(obj, type) (align_up(sizeof(obj), sizeof(type))/sizeof(type))
 
 // Native memory tracking level
+//
+// unknown: pre-init phase (before parsing NMT arguments)
+//
+// off:     after initialization - NMT confirmed off.
+//           - nothing is tracked
+//           - no malloc headers are used
+//
+// minimal: after shutdown - NMT had been on at some point but has been switched off
+//           - nothing is tracked
+//           - malloc headers are allocated but not initialized not used
+//
+// summary: after initialization with NativeMemoryTracking=summary - NMT in summary mode
+//           - category summaries per tag are tracked
+//           - thread stacks are tracked
+//           - malloc headers are used
+//           - malloc call site table is allocated and used
+//
+// detail:  after initialization with NativeMemoryTracking=detail - NMT in detail mode
+//           - category summaries per tag are tracked
+//           - malloc details per call site are tracked
+//           - virtual memory mapping info is tracked
+//           - thread stacks are tracked
+//           - malloc headers are used
+//           - malloc call site table is allocated and used
+//
+// Valid state transitions
+//
+// unknown ----> off
+//          |
+//          |--> summary --
+//          |              |
+//          |--> detail  --+--> minimal
+//
+// Please keep numerical values:
+// unknown < off < minimal < summary < detail
+//
 enum NMT_TrackingLevel {
-  NMT_unknown = 0xFF,
-  NMT_off     = 0x00,
-  NMT_minimal = 0x01,
-  NMT_summary = 0x02,
-  NMT_detail  = 0x03
+  NMT_unknown = 0,
+  NMT_off     = 1,
+  NMT_minimal = 2,
+  NMT_summary = 3,
+  NMT_detail  = 4
 };
 
 // Number of stack frames to capture. This is a
diff --git a/src/hotspot/share/services/nmtPreInitBuffer.cpp b/src/hotspot/share/services/nmtPreInitBuffer.cpp
new file mode 100644
index 00000000000..38610c50241
--- /dev/null
+++ b/src/hotspot/share/services/nmtPreInitBuffer.cpp
@@ -0,0 +1,210 @@
+/*
+ * Copyright (c) 2021 SAP SE. All rights reserved.
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "jvm_io.h"
+#include "logging/log.hpp"
+#include "services/memTracker.hpp"
+#include "services/nmtPreInitBuffer.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
+
+// Threadedness note:
+//
+// The NMTPreInitBuffer is guaranteed to be used only single threaded, since its
+// only used during VM initialization. However, that does not mean that its always
+// the same thread, since the thread loading the hotspot - which causes the
+// dynamic C++ initialization inside the hotspot to run, and allocations to happen
+// - may be a different thread from the one invoking CreateJavaVM.
+
+#if INCLUDE_NMT
+
+#include <stdlib.h> // for raw malloc
+
+const static size_t malloc_alignment = NOT_LP64(8) LP64_ONLY(16);
+
+// To be able to provide at least a primitive notion of realloc, we
+// need to know the block size, hence we need a small header.
+struct hdr {
+  size_t len;
+  size_t dummy;
+};
+
+STATIC_ASSERT(sizeof(hdr) == malloc_alignment);
+
+// Statistics
+static struct {
+  int allocs;
+  int reallocs; int inplace_reallocs;
+  int frees; int inplace_frees;
+} g_stats;
+
+static hdr* get_hdr(void* p) {
+  assert(is_aligned(p, malloc_alignment), "sanity");
+  return ((hdr*)p) - 1;
+}
+
+static size_t get_block_size(void* p) {
+  return get_hdr(p)->len;
+}
+
+uint8_t NMTPreInitBuffer::_buffer[buffer_size] = { 0 };
+uint8_t* NMTPreInitBuffer::_top = NULL;
+
+void NMTPreInitBuffer::initialize() {
+  assert(_top == NULL, "sanity");
+  _top = _buffer;
+}
+
+// Allocate s bytes from the preinit buffer. Can only be called before NMT initialization.
+// On buffer exhaustion, NMT is switched off and C-heap is returned instead (release);
+// in debug builds we assert.
+void* NMTPreInitBuffer::allocate(size_t size, MEMFLAGS flag) {
+
+  // Initialize on very first malloc. We need to do this on demand since we cannot
+  // rely on order of initialization.
+  if (!is_initialized()) {
+    initialize();
+  }
+
+  // We only allow this before NMT initialization.
+  assert(!MemTracker::is_initialized(), "Use only pre-NMT initialization");
+
+  // 0->1, and honor malloc alignment
+  const size_t inner_size = align_up(MAX2((size_t)1, size), malloc_alignment);
+  const size_t outer_size = inner_size + sizeof(hdr);
+
+  // Buffer exhausted?
+  if (_top + outer_size > end()) {
+    // On pre-init buffer exhaustion, disable NMT. This will end the pre-init phase, so
+    // we will not allocate from the pre-init buffer anymore. It will prevent NMT from
+    // being switched on later after argument parsing, but at least it allows the VM to
+    // continue.
+    MemTracker::initialize(NMT_off);
+    // Then, fulfill the allocation request from normal C-heap.
+    return os::malloc(size, flag);
+  }
+
+  // Allocate from top.
+  hdr* const new_hdr = (hdr*)_top;
+  new_hdr->len = inner_size;
+  uint8_t* const ret = (uint8_t*)(new_hdr + 1);
+  _top += outer_size;
+  assert(_top == ret + inner_size, "sanity");
+
+  DEBUG_ONLY(::memset(ret, 0x17, inner_size));
+
+  g_stats.allocs++;
+
+  return ret;
+}
+
+// Reallocate an allocation originally allocated from the preinit buffer within the preinit
+// buffer.  Can only be called before NMT initialization.
+// On buffer exhaustion, NMT is switched off and C-heap is returned instead (release);
+// in debug builds we assert.
+void* NMTPreInitBuffer::reallocate(void* old, size_t size, MEMFLAGS flag) {
+
+  assert(is_initialized(), "sanity");
+
+  // We only allow this before NMT initialization, and only from a single thread.
+  assert(!MemTracker::is_initialized(), "Use only pre-NMT initialization");
+
+  // The old block should not be NULL and be contained in the preinit buffer.
+  assert(contains(old), "sanity");
+
+  // Note: We don't bother to implement any optimizations here (eg in-place-realloc).
+  //  Chances of reallocs happening which would benefit are not high. If we start hurting
+  //  we can optimize (see also free below).
+  uint8_t* ret = (uint8_t*)allocate(size, flag);
+  if (size > 0 && old != NULL) {
+    size_t size_old = get_block_size(old);
+    if (size_old > 0) {
+      ::memcpy(ret, old, MIN2(size, size_old));
+    }
+    free(old);
+  }
+
+  g_stats.reallocs++;
+
+  return ret;
+}
+
+// Reallocate an allocation originally allocated from the preinit buffer into the regular
+// C-heap. Can only be called *after* NMT initialization.
+void* NMTPreInitBuffer::reallocate_to_c_heap(void* old, size_t size, MEMFLAGS flag) {
+
+  assert(is_initialized(), "sanity");
+
+  // This should only be called after NMT initialization.
+  assert(MemTracker::is_initialized(), "Use only post-NMT initialization");
+
+  // The old block should not be NULL and be contained in the preinit buffer.
+  assert(contains(old), "sanity");
+
+  uint8_t* ret = NEW_C_HEAP_ARRAY(uint8_t, size, flag);
+  if (size > 0 && old != NULL) {
+    size_t size_old = get_block_size(old);
+    if (size_old > 0) {
+      ::memcpy(ret, old, MIN2(size, size_old));
+    }
+  }
+
+  return ret;
+}
+
+// Attempts to free a block originally allocated from the preinit buffer (only rolls
+// back top allocation).
+void NMTPreInitBuffer::free(void* old) {
+
+  assert(is_initialized(), "sanity");
+
+  // The old block should not be NULL and be contained in the preinit buffer.
+  assert(contains(old), "sanity");
+
+  // Roll back top-of-arena-allocations...
+  const size_t old_size = get_block_size(old);
+  if (_top - old_size == (uint8_t*)old) {
+    _top = (uint8_t*)old - sizeof(hdr);
+    g_stats.inplace_frees++;
+  }
+
+  // ... otherwise just ignore. If we are really hurting, we may add some
+  // form of freeblock management; but lets keep things simple for now.
+  g_stats.frees++;
+}
+
+void NMTPreInitBuffer::print_state(outputStream* st) {
+  st->print_cr("buffer: " PTR_FORMAT ", used: " SIZE_FORMAT ", end: " PTR_FORMAT ", stats: %d/%d(%d)/%d(%d)",
+               p2i(_buffer), _top - _buffer, p2i(end()),
+               g_stats.allocs,
+               g_stats.reallocs, g_stats.inplace_reallocs,
+               g_stats.frees, g_stats.inplace_frees);
+}
+
+
+#endif // INCLUDE_NMT
diff --git a/src/hotspot/share/services/nmtPreInitBuffer.hpp b/src/hotspot/share/services/nmtPreInitBuffer.hpp
new file mode 100644
index 00000000000..fc65d4c2904
--- /dev/null
+++ b/src/hotspot/share/services/nmtPreInitBuffer.hpp
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2021 SAP SE. All rights reserved.
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_SERVICES_NMT_PREINIT_BUFFER_HPP
+#define SHARE_SERVICES_NMT_PREINIT_BUFFER_HPP
+
+
+
+#include "memory/allocation.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
+
+#if INCLUDE_NMT
+
+class outputStream;
+
+// The purpose of this class is to serve as a simple buffer for pre-initialization
+//  C-Heap allocations.
+class NMTPreInitBuffer : public AllStatic {
+
+  ATTRIBUTE_ALIGNED(16) static const size_t buffer_size = 64 * K;
+
+  static uint8_t _buffer[buffer_size];
+  static uint8_t* _top;
+  static const uint8_t* end() { return _buffer + buffer_size; }
+
+  static void initialize();
+  static bool is_initialized() { return _top != NULL; }
+
+public:
+
+  // Allocate s bytes from the preinit buffer. Can only be called before NMT initialization.
+  // On buffer exhaustion, NMT is switched off and C-heap is returned instead (release);
+  // in debug builds we assert.
+  static void* allocate(size_t s, MEMFLAGS flag);
+
+  // Reallocate an allocation originally allocated from the preinit buffer within the preinit
+  // buffer.  Can only be called before NMT initialization.
+  // On buffer exhaustion, NMT is switched off and C-heap is returned instead (release);
+  // in debug builds we assert.
+  static void* reallocate(void* old, size_t s, MEMFLAGS flag);
+
+  // Reallocate an allocation originally allocated from the preinit buffer into the regular
+  // C-heap. Can only be called *after* NMT initialization.
+  static void* reallocate_to_c_heap(void* old, size_t s, MEMFLAGS flag);
+
+  // Attempts to free a block originally allocated from the preinit buffer (only rolls
+  // back top allocation).
+  static void free(void* old);
+
+  // Needs to be fast
+  inline static bool contains(const void* p) {
+    assert(is_initialized(), "sanity");
+    return _buffer <= (uint8_t*)p &&
+           (uint8_t*)p < _buffer + sizeof(_buffer);
+  }
+
+  // print a string describing the current buffer state
+  static void print_state(outputStream* st);
+
+};
+
+#endif // INCLUDE_NMT
+
+#endif // SHARE_SERVICES_NMT_PREINIT_BUFFER_HPP
diff --git a/src/java.base/share/native/libjli/java.c b/src/java.base/share/native/libjli/java.c
index f4b702db7bb..614b3017aaa 100644
--- a/src/java.base/share/native/libjli/java.c
+++ b/src/java.base/share/native/libjli/java.c
@@ -804,8 +804,6 @@ CheckJvmType(int *pargc, char ***argv, jboolean speculative) {
 static void
 SetJvmEnvironment(int argc, char **argv) {
 
-    static const char*  NMT_Env_Name    = "NMT_LEVEL_";
-    const char* NMT_Arg_Name = IsJavaArgs() ? "-J-XX:NativeMemoryTracking=" : "-XX:NativeMemoryTracking=";
     int i;
     /* process only the launcher arguments */
     for (i = 0; i < argc; i++) {
@@ -831,46 +829,6 @@ SetJvmEnvironment(int argc, char **argv) {
                 return;
             }
         }
-        /*
-         * The following case checks for "-XX:NativeMemoryTracking=value".
-         * If value is non null, an environmental variable set to this value
-         * will be created to be used by the JVM.
-         * The argument is passed to the JVM, which will check validity.
-         * The JVM is responsible for removing the env variable.
-         */
-        if (JLI_StrCCmp(arg, NMT_Arg_Name) == 0) {
-            int retval;
-            // get what follows this parameter, include "="
-            size_t pnlen = JLI_StrLen(NMT_Arg_Name);
-            if (JLI_StrLen(arg) > pnlen) {
-                char* value = arg + pnlen;
-                size_t pbuflen = pnlen + JLI_StrLen(value) + 10; // 10 max pid digits
-
-                /*
-                 * ensures that malloc successful
-                 * DONT JLI_MemFree() pbuf.  JLI_PutEnv() uses system call
-                 *   that could store the address.
-                 */
-                char * pbuf = (char*)JLI_MemAlloc(pbuflen);
-
-                JLI_Snprintf(pbuf, pbuflen, "%s%d=%s", NMT_Env_Name, JLI_GetPid(), value);
-                retval = JLI_PutEnv(pbuf);
-                if (JLI_IsTraceLauncher()) {
-                    char* envName;
-                    char* envBuf;
-
-                    // ensures that malloc successful
-                    envName = (char*)JLI_MemAlloc(pbuflen);
-                    JLI_Snprintf(envName, pbuflen, "%s%d", NMT_Env_Name, JLI_GetPid());
-
-                    printf("TRACER_MARKER: NativeMemoryTracking: env var is %s\n",envName);
-                    printf("TRACER_MARKER: NativeMemoryTracking: putenv arg %s\n",pbuf);
-                    envBuf = getenv(envName);
-                    printf("TRACER_MARKER: NativeMemoryTracking: got value %s\n",envBuf);
-                    free(envName);
-                }
-            }
-        }
     }
 }
 
diff --git a/test/hotspot/gtest/metaspace/test_virtualspacenode.cpp b/test/hotspot/gtest/metaspace/test_virtualspacenode.cpp
index 344028cb646..ec44616d99e 100644
--- a/test/hotspot/gtest/metaspace/test_virtualspacenode.cpp
+++ b/test/hotspot/gtest/metaspace/test_virtualspacenode.cpp
@@ -33,6 +33,7 @@
 #include "memory/metaspace/metaspaceSettings.hpp"
 #include "memory/metaspace/virtualSpaceNode.hpp"
 #include "runtime/mutexLocker.hpp"
+#include "services/memTracker.hpp"
 #include "utilities/debug.hpp"
 //#define LOG_PLEASE
 #include "metaspaceGtestCommon.hpp"
@@ -338,6 +339,11 @@ public:
 
   void test_arbitrary_commits() {
 
+    // Disable test if NMT is on. Waiting for JDK-8263455 to be fixed.
+    if (MemTracker::tracking_level() > NMT_off) {
+      return;
+    }
+
     assert(_commit_limit >= _vs_word_size, "For this test no commit limit.");
 
     // Get a root chunk to have a committable region
diff --git a/test/hotspot/gtest/runtime/test_os.cpp b/test/hotspot/gtest/runtime/test_os.cpp
index 8bc1357bdb0..0dd24835143 100644
--- a/test/hotspot/gtest/runtime/test_os.cpp
+++ b/test/hotspot/gtest/runtime/test_os.cpp
@@ -26,6 +26,7 @@
 #include "memory/resourceArea.hpp"
 #include "runtime/os.hpp"
 #include "runtime/thread.hpp"
+#include "services/memTracker.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/macros.hpp"
 #include "utilities/ostream.hpp"
@@ -417,6 +418,15 @@ TEST_VM(os, DISABLED_release_multi_mappings) {
 #else
 TEST_VM(os, release_multi_mappings) {
 #endif
+
+  // With NMT enabled, this will trigger JDK-8263464.
+  // (NMT assumes that we only release regions or parts of regions, but cannot deal
+  //  with us releasing multiple regions.)
+  // For now disable the test if NMT=on.
+/*  if (MemTracker::tracking_level() > NMT_off) {
+    return;
+  }
+*/
   // Test that we can release an area created with multiple reservation calls
   const size_t stripe_len = 4 * M;
   const int num_stripes = 4;
diff --git a/test/jdk/tools/launcher/TestSpecialArgs.java b/test/jdk/tools/launcher/TestSpecialArgs.java
index a3953093a08..4f090e0ff65 100644
--- a/test/jdk/tools/launcher/TestSpecialArgs.java
+++ b/test/jdk/tools/launcher/TestSpecialArgs.java
@@ -111,94 +111,6 @@ public class TestSpecialArgs extends TestHelper {
         }
     }
 
-    @Test
-    void testNativeMemoryTracking() {
-        final Map<String, String> envMap = new HashMap<>();
-        envMap.put("_JAVA_LAUNCHER_DEBUG", "true");
-        TestResult tr;
-        /*
-         * test argument : -XX:NativeMemoryTracking=value
-         * A JVM flag, comsumed by the JVM, but requiring launcher
-         * to set an environmental variable if and only if value is supplied.
-         * Test and order:
-         * 1) execute with valid parameter: -XX:NativeMemoryTracking=MyValue
-         *    a) check for correct env variable name: "NMT_LEVEL_" + pid
-         *    b) check that "MyValue" was found in local env.
-         * 2) execute with invalid parameter: -XX:NativeMemoryTracking=
-         *    !) Won't find "NativeMemoryTracking:"
-         *       Code to create env variable not executed.
-         * 3) execute with invalid parameter: -XX:NativeMemoryTracking
-         *    !) Won't find "NativeMemoryTracking:"
-         *       Code to create env variable not executed.
-         * 4) give and invalid value and check to make sure JVM commented
-         */
-        String envVarPidString = "TRACER_MARKER: NativeMemoryTracking: env var is NMT_LEVEL_";
-        String NMT_Option_Value = "off";
-        String myClassName = "helloworld";
-
-        // === Run the tests ===
-        // ---Test 1a
-        tr = doExec(envMap, javaCmd, "-XX:NativeMemoryTracking=" + NMT_Option_Value,
-                "-version");
-
-        // get the PID from the env var we set for the JVM
-        String envVarPid = null;
-        for (String line : tr.testOutput) {
-            if (line.contains(envVarPidString)) {
-                int sindex = envVarPidString.length();
-                envVarPid = line.substring(sindex);
-                break;
-            }
-        }
-        // did we find envVarPid?
-        if (envVarPid == null) {
-            System.out.println(tr);
-            throw new RuntimeException("Error: failed to find env Var Pid in tracking info");
-        }
-        // we think we found the pid string.  min test, not "".
-        if (envVarPid.length() < 1) {
-            System.out.println(tr);
-            throw new RuntimeException("Error: env Var Pid in tracking info is empty string");
-        }
-
-        // --- Test 1b
-        if (!tr.contains("NativeMemoryTracking: got value " + NMT_Option_Value)) {
-            System.out.println(tr);
-            throw new RuntimeException("Error: Valid param failed to set env variable");
-        }
-
-        // --- Test 2
-        tr = doExec(envMap, javaCmd, "-XX:NativeMemoryTracking=",
-                "-version");
-        if (tr.contains("NativeMemoryTracking:")) {
-            System.out.println(tr);
-            throw new RuntimeException("Error: invalid param caused env variable to be erroneously created");
-        }
-        if (!tr.contains("Syntax error, expecting -XX:NativeMemoryTracking=")) {
-            System.out.println(tr);
-            throw new RuntimeException("Error: invalid param not checked by JVM");
-        }
-
-        // --- Test 3
-        tr = doExec(envMap, javaCmd, "-XX:NativeMemoryTracking",
-                "-version");
-        if (tr.contains("NativeMemoryTracking:")) {
-            System.out.println(tr);
-            throw new RuntimeException("Error: invalid param caused env variable to be erroneously created");
-        }
-        if (!tr.contains("Syntax error, expecting -XX:NativeMemoryTracking=")) {
-            System.out.println(tr);
-            throw new RuntimeException("Error: invalid param not checked by JVM");
-        }
-        // --- Test 4
-        tr = doExec(envMap, javaCmd, "-XX:NativeMemoryTracking=BADVALUE",
-                "-version");
-        if (!tr.contains("expecting -XX:NativeMemoryTracking")) {
-            System.out.println(tr);
-            throw new RuntimeException("Error: invalid param did not get JVM Syntax error message");
-        }
-    }
-
     @Test
     void testNMArgumentProcessing() throws FileNotFoundException {
         TestResult tr;