--- old/src/hotspot/os/bsd/os_bsd.cpp	2019-09-21 06:24:55.549962624 -0700
+++ new/src/hotspot/os/bsd/os_bsd.cpp	2019-09-21 06:24:55.189962637 -0700
@@ -2026,6 +2026,10 @@
   return 0;
 }
 
+int os::numa_get_address_id(uintptr_t addr) {
+  return 0;
+}
+
 bool os::get_page_info(char *start, page_info* info) {
   return false;
 }
--- old/src/hotspot/os/linux/os_linux.cpp	2019-09-21 06:24:56.529962590 -0700
+++ new/src/hotspot/os/linux/os_linux.cpp	2019-09-21 06:24:56.197962602 -0700
@@ -3011,6 +3011,24 @@
   return 0;
 }
 
+int os::numa_get_address_id(uintptr_t addr) {
+#ifndef MPOL_F_NODE
+#define MPOL_F_NODE     (1<<0)  // Return next IL mode instead of node mask
+#endif
+
+#ifndef MPOL_F_ADDR
+#define MPOL_F_ADDR     (1<<1)  // Look up VMA using address
+#endif
+  
+  uint32_t id = (uint32_t)-1;
+
+  if (syscall(SYS_get_mempolicy, &id, NULL, 0, addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) {
+    warning("Failed to get numa id at " PTR_FORMAT " with errno=%d", p2i((void*)addr), errno);
+    return os::InvalidId;
+  }
+  return id;
+}
+
 int os::Linux::get_existing_num_nodes() {
   int node;
   int highest_node_number = Linux::numa_max_node();
--- old/src/hotspot/os/solaris/os_solaris.cpp	2019-09-21 06:24:58.561962519 -0700
+++ new/src/hotspot/os/solaris/os_solaris.cpp	2019-09-21 06:24:58.201962532 -0700
@@ -2233,7 +2233,7 @@
   char *res = Solaris::mmap_chunk(addr, size, MAP_PRIVATE|MAP_FIXED, prot);
   if (res != NULL) {
     if (UseNUMAInterleaving) {
-      numa_make_global(addr, bytes);
+        numa_make_global(addr, bytes);
     }
     return 0;
   }
@@ -2428,6 +2428,10 @@
   return ids[os::random() % r];
 }
 
+int os::numa_get_address_id(uintptr_t addr) {
+  return 0;
+}
+
 // Request information about the page.
 bool os::get_page_info(char *start, page_info* info) {
   const uint_t info_types[] = { MEMINFO_VLGRP, MEMINFO_VPAGESIZE };
--- old/src/hotspot/os/windows/os_windows.cpp	2019-09-21 06:24:59.641962482 -0700
+++ new/src/hotspot/os/windows/os_windows.cpp	2019-09-21 06:24:59.289962494 -0700
@@ -3456,6 +3456,8 @@
   }
 }
 
+int os::numa_get_address_id(uintptr_t addr) { return 0; }
+
 bool os::get_page_info(char *start, page_info* info) {
   return false;
 }
--- old/src/hotspot/share/gc/g1/g1AllocRegion.cpp	2019-09-21 06:25:00.801962441 -0700
+++ new/src/hotspot/share/gc/g1/g1AllocRegion.cpp	2019-09-21 06:25:00.429962454 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -250,17 +250,19 @@
 #endif // PRODUCT
 
 G1AllocRegion::G1AllocRegion(const char* name,
-                             bool bot_updates)
+                             bool bot_updates,
+                             uint node_index)
   : _alloc_region(NULL),
     _count(0),
     _used_bytes_before(0),
     _bot_updates(bot_updates),
-    _name(name)
+    _name(name),
+    _node_index(node_index)
  { }
 
 HeapRegion* MutatorAllocRegion::allocate_new_region(size_t word_size,
                                                     bool force) {
-  return _g1h->new_mutator_alloc_region(word_size, force);
+  return _g1h->new_mutator_alloc_region(word_size, force, _node_index);
 }
 
 void MutatorAllocRegion::retire_region(HeapRegion* alloc_region,
--- old/src/hotspot/share/gc/g1/g1AllocRegion.hpp	2019-09-21 06:25:01.813962406 -0700
+++ new/src/hotspot/share/gc/g1/g1AllocRegion.hpp	2019-09-21 06:25:01.457962418 -0700
@@ -28,6 +28,7 @@
 #include "gc/g1/heapRegion.hpp"
 #include "gc/g1/g1EvacStats.hpp"
 #include "gc/g1/g1HeapRegionAttr.hpp"
+#include "gc/g1/g1MemoryNodeManager.hpp"
 
 class G1CollectedHeap;
 
@@ -38,7 +39,7 @@
 // and a lock will need to be taken when the active region needs to be
 // replaced.
 
-class G1AllocRegion {
+class G1AllocRegion : public CHeapObj<mtGC> {
 
 private:
   // The active allocating region we are currently allocating out
@@ -91,6 +92,9 @@
   HeapWord* new_alloc_region_and_allocate(size_t word_size, bool force);
 
 protected:
+  // The memory node index this allocation region belongs to.
+  uint _node_index;
+
   // Reset the alloc region to point a the dummy region.
   void reset_alloc_region();
 
@@ -131,7 +135,7 @@
   virtual void retire_region(HeapRegion* alloc_region,
                              size_t allocated_bytes) = 0;
 
-  G1AllocRegion(const char* name, bool bot_updates);
+  G1AllocRegion(const char* name, bool bot_updates, uint node_index);
 
 public:
   static void setup(G1CollectedHeap* g1h, HeapRegion* dummy_region);
@@ -220,8 +224,8 @@
   virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes);
   virtual size_t retire(bool fill_up);
 public:
-  MutatorAllocRegion()
-    : G1AllocRegion("Mutator Alloc Region", false /* bot_updates */),
+  MutatorAllocRegion(uint node_index)
+    : G1AllocRegion("Mutator Alloc Region", false /* bot_updates */, node_index),
       _wasted_bytes(0),
       _retained_alloc_region(NULL) { }
 
@@ -245,6 +249,7 @@
 
   virtual void init();
 };
+
 // Common base class for allocation regions used during GC.
 class G1GCAllocRegion : public G1AllocRegion {
 protected:
@@ -256,8 +261,9 @@
 
   virtual size_t retire(bool fill_up);
 
-  G1GCAllocRegion(const char* name, bool bot_updates, G1EvacStats* stats, G1HeapRegionAttr::region_type_t purpose)
-  : G1AllocRegion(name, bot_updates), _stats(stats), _purpose(purpose) {
+  G1GCAllocRegion(const char* name, bool bot_updates, G1EvacStats* stats,
+                  G1HeapRegionAttr::region_type_t purpose, uint node_index = G1MemoryNodeManager::AnyNodeIndex)
+  : G1AllocRegion(name, bot_updates, node_index), _stats(stats), _purpose(purpose) {
     assert(stats != NULL, "Must pass non-NULL PLAB statistics");
   }
 };
--- old/src/hotspot/share/gc/g1/g1Allocator.cpp	2019-09-21 06:25:02.777962372 -0700
+++ new/src/hotspot/share/gc/g1/g1Allocator.cpp	2019-09-21 06:25:02.449962384 -0700
@@ -38,20 +38,44 @@
   _g1h(heap),
   _survivor_is_full(false),
   _old_is_full(false),
-  _mutator_alloc_region(),
+  _num_alloc_region(heap->mem_node_mgr()->num_active_nodes()),
+  _mutator_alloc_region(NULL),
   _survivor_gc_alloc_region(heap->alloc_buffer_stats(G1HeapRegionAttr::Young)),
   _old_gc_alloc_region(heap->alloc_buffer_stats(G1HeapRegionAttr::Old)),
   _retained_old_gc_alloc_region(NULL) {
+
+  _mutator_alloc_region = NEW_C_HEAP_ARRAY(MutatorAllocRegion, _num_alloc_region, mtGC);
+  for (uint i = 0; i < _num_alloc_region; i++) {
+    ::new(_mutator_alloc_region + i) MutatorAllocRegion(i);
+  }
+}
+
+G1Allocator::~G1Allocator() {
+  for (uint i = 0; i < _num_alloc_region; i++) {
+    _mutator_alloc_region[i].~MutatorAllocRegion();
+  }
+  FREE_C_HEAP_ARRAY(MutatorAllocRegion, _mutator_alloc_region);
 }
 
+#ifdef ASSERT
+bool G1Allocator::has_mutator_alloc_region() {
+  uint node_index = _g1h->mem_node_mgr()->index_of_current_thread();
+  return mutator_alloc_region(node_index)->get() != NULL;
+}
+#endif
+
 void G1Allocator::init_mutator_alloc_region() {
-  assert(_mutator_alloc_region.get() == NULL, "pre-condition");
-  _mutator_alloc_region.init();
+  for (uint i = 0; i < _num_alloc_region; i++) {
+    assert(mutator_alloc_region(i)->get() == NULL, "pre-condition");
+    mutator_alloc_region(i)->init();
+  }
 }
 
 void G1Allocator::release_mutator_alloc_region() {
-  _mutator_alloc_region.release();
-  assert(_mutator_alloc_region.get() == NULL, "post-condition");
+  for (uint i = 0; i < _num_alloc_region; i++) {
+    mutator_alloc_region(i)->release();
+    assert(mutator_alloc_region(i)->get() == NULL, "post-condition");
+  }
 }
 
 bool G1Allocator::is_retained_old_region(HeapRegion* hr) {
@@ -146,7 +170,8 @@
   // since we can't allow tlabs to grow big enough to accommodate
   // humongous objects.
 
-  HeapRegion* hr = mutator_alloc_region()->get();
+  uint node_index = _g1h->mem_node_mgr()->index_of_current_thread();
+  HeapRegion* hr = mutator_alloc_region(node_index)->get();
   size_t max_tlab = _g1h->max_tlab_size() * wordSize;
   if (hr == NULL) {
     return max_tlab;
@@ -156,8 +181,11 @@
 }
 
 size_t G1Allocator::used_in_alloc_regions() {
-  assert(Heap_lock->owner() != NULL, "Should be owned on this thread's behalf.");
-  return mutator_alloc_region()->used_in_alloc_regions();
+  size_t used = 0;
+  for (uint i = 0; i < _num_alloc_region; i++) {
+    used += mutator_alloc_region(i)->used_in_alloc_regions();
+  }
+  return used;
 }
 
 
--- old/src/hotspot/share/gc/g1/g1Allocator.hpp	2019-09-21 06:25:03.801962337 -0700
+++ new/src/hotspot/share/gc/g1/g1Allocator.hpp	2019-09-21 06:25:03.433962349 -0700
@@ -27,6 +27,7 @@
 
 #include "gc/g1/g1AllocRegion.hpp"
 #include "gc/g1/g1HeapRegionAttr.hpp"
+#include "gc/g1/g1NUMA.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/plab.hpp"
 
@@ -44,8 +45,11 @@
   bool _survivor_is_full;
   bool _old_is_full;
 
+  // The number of MutatorAllocRegions used, one per memory node.
+  size_t _num_alloc_region;
+
   // Alloc region used to satisfy mutator allocation requests.
-  MutatorAllocRegion _mutator_alloc_region;
+  MutatorAllocRegion* _mutator_alloc_region;
 
   // Alloc region used to satisfy allocation requests by the GC for
   // survivor objects.
@@ -68,25 +72,26 @@
                                  HeapRegion** retained);
 
   // Accessors to the allocation regions.
-  inline MutatorAllocRegion* mutator_alloc_region();
+  inline MutatorAllocRegion* mutator_alloc_region(uint node_index);
   inline SurvivorGCAllocRegion* survivor_gc_alloc_region();
   inline OldGCAllocRegion* old_gc_alloc_region();
 
   // Allocation attempt during GC for a survivor object / PLAB.
   HeapWord* survivor_attempt_allocation(size_t min_word_size,
-                                               size_t desired_word_size,
-                                               size_t* actual_word_size);
+                                        size_t desired_word_size,
+                                        size_t* actual_word_size);
 
   // Allocation attempt during GC for an old object / PLAB.
   HeapWord* old_attempt_allocation(size_t min_word_size,
-                                          size_t desired_word_size,
-                                          size_t* actual_word_size);
+                                   size_t desired_word_size,
+                                   size_t* actual_word_size);
 public:
   G1Allocator(G1CollectedHeap* heap);
+  ~G1Allocator();
 
 #ifdef ASSERT
   // Do we currently have an active mutator region to allocate into?
-  bool has_mutator_alloc_region() { return mutator_alloc_region()->get() != NULL; }
+  bool has_mutator_alloc_region();
 #endif
 
   void init_mutator_alloc_region();
--- old/src/hotspot/share/gc/g1/g1Allocator.inline.hpp	2019-09-21 06:25:04.849962300 -0700
+++ new/src/hotspot/share/gc/g1/g1Allocator.inline.hpp	2019-09-21 06:25:04.489962313 -0700
@@ -30,8 +30,9 @@
 #include "gc/shared/plab.inline.hpp"
 #include "memory/universe.hpp"
 
-inline MutatorAllocRegion* G1Allocator::mutator_alloc_region() {
-  return &_mutator_alloc_region;
+inline MutatorAllocRegion* G1Allocator::mutator_alloc_region(uint node_index) {
+  assert(_g1h->mem_node_mgr()->is_valid_node_index(node_index), "Invariant, index %u", node_index);
+  return &_mutator_alloc_region[node_index];
 }
 
 inline SurvivorGCAllocRegion* G1Allocator::survivor_gc_alloc_region() {
@@ -45,22 +46,25 @@
 inline HeapWord* G1Allocator::attempt_allocation(size_t min_word_size,
                                                  size_t desired_word_size,
                                                  size_t* actual_word_size) {
-  HeapWord* result = mutator_alloc_region()->attempt_retained_allocation(min_word_size, desired_word_size, actual_word_size);
+  uint node_index = _g1h->mem_node_mgr()->index_of_current_thread();
+  HeapWord* result = mutator_alloc_region(node_index)->attempt_retained_allocation(min_word_size, desired_word_size, actual_word_size);
   if (result != NULL) {
     return result;
   }
-  return mutator_alloc_region()->attempt_allocation(min_word_size, desired_word_size, actual_word_size);
+  return mutator_alloc_region(node_index)->attempt_allocation(min_word_size, desired_word_size, actual_word_size);
 }
 
 inline HeapWord* G1Allocator::attempt_allocation_locked(size_t word_size) {
-  HeapWord* result = mutator_alloc_region()->attempt_allocation_locked(word_size);
-  assert(result != NULL || mutator_alloc_region()->get() == NULL,
-         "Must not have a mutator alloc region if there is no memory, but is " PTR_FORMAT, p2i(mutator_alloc_region()->get()));
+  uint node_index = _g1h->mem_node_mgr()->index_of_current_thread();
+  HeapWord* result = mutator_alloc_region(node_index)->attempt_allocation_locked(word_size);
+  assert(result != NULL || mutator_alloc_region(node_index)->get() == NULL,
+         "Must not have a mutator alloc region if there is no memory, but is " PTR_FORMAT, p2i(mutator_alloc_region(node_index)->get()));
   return result;
 }
 
 inline HeapWord* G1Allocator::attempt_allocation_force(size_t word_size) {
-  return mutator_alloc_region()->attempt_allocation_force(word_size);
+  uint node_index = _g1h->mem_node_mgr()->index_of_current_thread();
+  return mutator_alloc_region(node_index)->attempt_allocation_force(word_size);
 }
 
 inline PLAB* G1PLABAllocator::alloc_buffer(G1HeapRegionAttr dest) {
--- old/src/hotspot/share/gc/g1/g1Arguments.cpp	2019-09-21 06:25:05.865962265 -0700
+++ new/src/hotspot/share/gc/g1/g1Arguments.cpp	2019-09-21 06:25:05.509962277 -0700
@@ -158,6 +158,16 @@
     FLAG_SET_DEFAULT(ParallelRefProcEnabled, true);
   }
 
+  if (UseNUMA) {
+    if (FLAG_IS_DEFAULT(AlwaysPreTouch)) {
+      FLAG_SET_DEFAULT(AlwaysPreTouch, true);
+    }
+    if (!AlwaysPreTouch && FLAG_IS_CMDLINE(AlwaysPreTouch)) {
+      warning("Disabling AlwaysPreTouch is incompatible with UseNUMA. Disabling UseNUMA.");
+      FLAG_SET_ERGO(UseNUMA, false);
+    }
+  }
+
   log_trace(gc)("MarkStackSize: %uk  MarkStackSizeMax: %uk", (unsigned int) (MarkStackSize / K), (uint) (MarkStackSizeMax / K));
 
   // By default do not let the target stack size to be more than 1/4 of the entries
--- old/src/hotspot/share/gc/g1/g1CollectedHeap.cpp	2019-09-21 06:25:06.797962232 -0700
+++ new/src/hotspot/share/gc/g1/g1CollectedHeap.cpp	2019-09-21 06:25:06.453962244 -0700
@@ -169,12 +169,15 @@
 
 // Private methods.
 
-HeapRegion* G1CollectedHeap::new_region(size_t word_size, HeapRegionType type, bool do_expand) {
+HeapRegion* G1CollectedHeap::new_region(size_t word_size,
+                                        HeapRegionType type,
+                                        bool do_expand,
+                                        uint node_index) {
   assert(!is_humongous(word_size) || word_size <= HeapRegion::GrainWords,
          "the only time we use this to allocate a humongous region is "
          "when we are allocating a single humongous region");
 
-  HeapRegion* res = _hrm->allocate_free_region(type);
+  HeapRegion* res = _hrm->allocate_free_region(type, node_index);
 
   if (res == NULL && do_expand && _expand_heap_after_alloc_failure) {
     // Currently, only attempts to allocate GC alloc regions set
@@ -186,12 +189,12 @@
     log_debug(gc, ergo, heap)("Attempt heap expansion (region allocation request failed). Allocation request: " SIZE_FORMAT "B",
                               word_size * HeapWordSize);
 
-    if (expand(word_size * HeapWordSize)) {
+    if (expand(word_size * HeapWordSize, node_index)) {
       // Given that expand() succeeded in expanding the heap, and we
       // always expand the heap by an amount aligned to the heap
       // region size, the free list should in theory not be empty.
       // In either case allocate_free_region() will check for NULL.
-      res = _hrm->allocate_free_region(type);
+      res = _hrm->allocate_free_region(type, node_index);
     } else {
       _expand_heap_after_alloc_failure = false;
     }
@@ -362,7 +365,7 @@
       log_debug(gc, ergo, heap)("Attempt heap expansion (humongous allocation request failed). Allocation request: " SIZE_FORMAT "B",
                                     word_size * HeapWordSize);
 
-      _hrm->expand_at(first, obj_regions, workers());
+      _hrm->expand_at(first, obj_regions, G1MemoryNodeManager::AnyNodeIndex, workers());
       policy()->record_new_heap_size(num_regions());
 
 #ifdef ASSERT
@@ -1223,7 +1226,7 @@
                               "min_desired_capacity: " SIZE_FORMAT "B (" UINTX_FORMAT " %%)",
                               capacity_after_gc, used_after_gc, used(), minimum_desired_capacity, MinHeapFreeRatio);
 
-    expand(expand_bytes, _workers);
+    expand(expand_bytes, G1MemoryNodeManager::AnyNodeIndex, _workers);
 
     // No expansion, now see if we want to shrink
   } else if (capacity_after_gc > maximum_desired_capacity) {
@@ -1334,7 +1337,7 @@
                             word_size * HeapWordSize);
 
 
-  if (expand(expand_bytes, _workers)) {
+  if (expand(expand_bytes, G1MemoryNodeManager::AnyNodeIndex, _workers)) {
     _hrm->verify_optional();
     _verifier->verify_region_sets_optional();
     return attempt_allocation_at_safepoint(word_size,
@@ -1343,7 +1346,7 @@
   return NULL;
 }
 
-bool G1CollectedHeap::expand(size_t expand_bytes, WorkGang* pretouch_workers, double* expand_time_ms) {
+bool G1CollectedHeap::expand(size_t expand_bytes, uint node_index, WorkGang* pretouch_workers, double* expand_time_ms) {
   size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);
   aligned_expand_bytes = align_up(aligned_expand_bytes,
                                        HeapRegion::GrainBytes);
@@ -1360,7 +1363,7 @@
   uint regions_to_expand = (uint)(aligned_expand_bytes / HeapRegion::GrainBytes);
   assert(regions_to_expand > 0, "Must expand by at least one region");
 
-  uint expanded_by = _hrm->expand_by(regions_to_expand, pretouch_workers);
+  uint expanded_by = _hrm->expand_by(regions_to_expand, node_index, pretouch_workers);
   if (expand_time_ms != NULL) {
     *expand_time_ms = (os::elapsedTime() - expand_heap_start_time_sec) * MILLIUNITS;
   }
@@ -1393,7 +1396,6 @@
   uint num_regions_removed = _hrm->shrink_by(num_regions_to_remove);
   size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes;
 
-
   log_debug(gc, ergo, heap)("Shrink the heap. requested shrinking amount: " SIZE_FORMAT "B aligned shrinking amount: " SIZE_FORMAT "B attempted shrinking amount: " SIZE_FORMAT "B",
                             shrink_bytes, aligned_shrink_bytes, shrunk_bytes);
   if (num_regions_removed > 0) {
@@ -1495,6 +1497,7 @@
   _humongous_set("Humongous Region Set", new HumongousRegionSetChecker()),
   _bot(NULL),
   _listener(),
+  _mem_node_mgr(G1MemoryNodeManager::create()),
   _hrm(NULL),
   _allocator(NULL),
   _verifier(NULL),
@@ -1778,6 +1781,8 @@
   }
   _workers->initialize_workers();
 
+  _mem_node_mgr->set_page_size(page_size);
+
   // Create the G1ConcurrentMark data structure and thread.
   // (Must do this late, so that "max_regions" is defined.)
   _cm = new G1ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage);
@@ -1788,7 +1793,7 @@
   _cm_thread = _cm->cm_thread();
 
   // Now expand into the initial heap size.
-  if (!expand(init_byte_size, _workers)) {
+  if (!expand(init_byte_size, G1MemoryNodeManager::AnyNodeIndex, _workers)) {
     vm_shutdown_during_initialization("Failed to allocate initial heap.");
     return JNI_ENOMEM;
   }
@@ -2886,7 +2891,7 @@
     // No need for an ergo logging here,
     // expansion_amount() does this when it returns a value > 0.
     double expand_ms;
-    if (!expand(expand_bytes, _workers, &expand_ms)) {
+    if (!expand(expand_bytes, G1MemoryNodeManager::AnyNodeIndex, _workers, &expand_ms)) {
       // We failed to expand the heap. Cannot do anything about it.
     }
     phase_times()->record_expand_heap_time(expand_ms);
@@ -4547,13 +4552,15 @@
 // Methods for the mutator alloc region
 
 HeapRegion* G1CollectedHeap::new_mutator_alloc_region(size_t word_size,
-                                                      bool force) {
+                                                      bool force,
+                                                      uint node_index) {
   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
   bool should_allocate = policy()->should_allocate_mutator_region();
   if (force || should_allocate) {
     HeapRegion* new_alloc_region = new_region(word_size,
                                               HeapRegionType::Eden,
-                                              false /* do_expand */);
+                                              false /* do_expand */,
+                                              node_index);
     if (new_alloc_region != NULL) {
       set_region_short_lived_locked(new_alloc_region);
       _hr_printer.alloc(new_alloc_region, !should_allocate);
--- old/src/hotspot/share/gc/g1/g1CollectedHeap.hpp	2019-09-21 06:25:07.989962191 -0700
+++ new/src/hotspot/share/gc/g1/g1CollectedHeap.hpp	2019-09-21 06:25:07.629962203 -0700
@@ -40,6 +40,7 @@
 #include "gc/g1/g1HeapVerifier.hpp"
 #include "gc/g1/g1HRPrinter.hpp"
 #include "gc/g1/g1HeapRegionAttr.hpp"
+#include "gc/g1/g1MemoryNodeManager.hpp"
 #include "gc/g1/g1MonitoringSupport.hpp"
 #include "gc/g1/g1RedirtyCardsQueue.hpp"
 #include "gc/g1/g1SurvivorRegions.hpp"
@@ -192,6 +193,9 @@
   // Callback for region mapping changed events.
   G1RegionMappingChangedListener _listener;
 
+  // Manages single or multi node memory.
+  G1MemoryNodeManager* _mem_node_mgr;
+
   // The sequence of all heap regions in the heap.
   HeapRegionManager* _hrm;
 
@@ -388,7 +392,10 @@
   // attempt to expand the heap if necessary to satisfy the allocation
   // request. 'type' takes the type of region to be allocated. (Use constants
   // Old, Eden, Humongous, Survivor defined in HeapRegionType.)
-  HeapRegion* new_region(size_t word_size, HeapRegionType type, bool do_expand);
+  HeapRegion* new_region(size_t word_size,
+                         HeapRegionType type,
+                         bool do_expand,
+                         uint node_index = G1MemoryNodeManager::AnyNodeIndex);
 
   // Initialize a contiguous set of free regions of length num_regions
   // and starting at index first so that they appear as a single
@@ -463,7 +470,7 @@
   // These methods are the "callbacks" from the G1AllocRegion class.
 
   // For mutator alloc regions.
-  HeapRegion* new_mutator_alloc_region(size_t word_size, bool force);
+  HeapRegion* new_mutator_alloc_region(size_t word_size, bool force, uint node_index);
   void retire_mutator_alloc_region(HeapRegion* alloc_region,
                                    size_t allocated_bytes);
 
@@ -548,11 +555,13 @@
 
   void resize_heap_if_necessary();
 
+  G1MemoryNodeManager* mem_node_mgr() const { return _mem_node_mgr; }
+
   // Expand the garbage-first heap by at least the given size (in bytes!).
   // Returns true if the heap was expanded by the requested amount;
   // false otherwise.
   // (Rounds up to a HeapRegion boundary.)
-  bool expand(size_t expand_bytes, WorkGang* pretouch_workers = NULL, double* expand_time_ms = NULL);
+  bool expand(size_t expand_bytes, uint node_index, WorkGang* pretouch_workers = NULL, double* expand_time_ms = NULL);
 
   // Returns the PLAB statistics for a given destination.
   inline G1EvacStats* alloc_buffer_stats(G1HeapRegionAttr dest);
@@ -928,7 +937,6 @@
 
   G1CMSubjectToDiscoveryClosure _is_subject_to_discovery_cm;
 public:
-
   RefToScanQueue *task_queue(uint i) const;
 
   uint num_task_queues() const;
--- old/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp	2019-09-21 06:25:09.121962151 -0700
+++ new/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp	2019-09-21 06:25:08.757962164 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/g1/g1NUMA.inline.hpp"
 #include "gc/g1/g1PageBasedVirtualSpace.hpp"
 #include "gc/shared/workgroup.hpp"
 #include "oops/markWord.hpp"
@@ -33,13 +34,13 @@
 #include "utilities/align.hpp"
 #include "utilities/bitMap.inline.hpp"
 
-G1PageBasedVirtualSpace::G1PageBasedVirtualSpace(ReservedSpace rs, size_t used_size, size_t page_size) :
+G1PageBasedVirtualSpace::G1PageBasedVirtualSpace(ReservedSpace rs, size_t used_size, size_t page_size, MemoryType type) :
   _low_boundary(NULL), _high_boundary(NULL), _tail_size(0), _page_size(0),
-  _committed(mtGC), _dirty(mtGC), _special(false), _executable(false) {
-  initialize_with_page_size(rs, used_size, page_size);
+  _committed(mtGC), _dirty(mtGC), _special(false), _executable(false), _numa(NULL) {
+  initialize_with_page_size(rs, used_size, page_size, type);
 }
 
-void G1PageBasedVirtualSpace::initialize_with_page_size(ReservedSpace rs, size_t used_size, size_t page_size) {
+void G1PageBasedVirtualSpace::initialize_with_page_size(ReservedSpace rs, size_t used_size, size_t page_size, MemoryType type) {
   guarantee(rs.is_reserved(), "Given reserved space must have been reserved already.");
 
   vmassert(_low_boundary == NULL, "VirtualSpace already initialized");
@@ -70,6 +71,13 @@
   }
 
   _tail_size = used_size % _page_size;
+
+  // Set _numa only if:
+  //   1) This space is for java heap.
+  //   2) There are multiple memory nodes because some OSes allow enabling UseNUMA.
+  if (type == mtJavaHeap && G1MemoryNodeManager::mgr()->num_active_nodes() > 1) {
+    _numa = G1NUMA::numa();
+  }
 }
 
 G1PageBasedVirtualSpace::~G1PageBasedVirtualSpace() {
@@ -81,6 +89,7 @@
   _executable             = false;
   _page_size              = 0;
   _tail_size              = 0;
+  _numa                   = NULL;
 }
 
 size_t G1PageBasedVirtualSpace::committed_size() const {
@@ -189,7 +198,7 @@
   os::pretouch_memory(page_start(start_page), bounded_end_addr(end_page), _page_size);
 }
 
-bool G1PageBasedVirtualSpace::commit(size_t start_page, size_t size_in_pages) {
+bool G1PageBasedVirtualSpace::commit(size_t start_page, size_t size_in_pages, uint node_index) {
   // We need to make sure to commit all pages covered by the given area.
   guarantee(is_area_uncommitted(start_page, size_in_pages), "Specified area is not uncommitted");
 
@@ -207,6 +216,12 @@
   }
   _committed.set_range(start_page, end_page);
 
+  if (_numa != NULL) {
+    char* start_addr = page_start(start_page);
+    size_t size_in_bytes = size_in_pages * _page_size;
+    _numa->touch_memory((address)start_addr, size_in_bytes, node_index);
+  }
+
   return zero_filled;
 }
 
--- old/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.hpp	2019-09-21 06:25:10.189962114 -0700
+++ new/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.hpp	2019-09-21 06:25:09.857962126 -0700
@@ -30,6 +30,7 @@
 #include "utilities/align.hpp"
 #include "utilities/bitMap.hpp"
 
+class G1NUMA;
 class WorkGang;
 
 // Virtual space management helper for a virtual space with an OS page allocation
@@ -74,6 +75,8 @@
   // Indicates whether the committed space should be executable.
   bool _executable;
 
+  G1NUMA* _numa;
+
   // Helper function for committing memory. Commit the given memory range by using
   // _page_size pages as much as possible and the remainder with small sized pages.
   void commit_internal(size_t start_page, size_t end_page);
@@ -109,22 +112,22 @@
   // Returns true if the entire area is not backed by committed memory.
   bool is_area_uncommitted(size_t start_page, size_t size_in_pages) const;
 
-  void initialize_with_page_size(ReservedSpace rs, size_t used_size, size_t page_size);
+  void initialize_with_page_size(ReservedSpace rs, size_t used_size, size_t page_size, MemoryType type);
  public:
 
   // Commit the given area of pages starting at start being size_in_pages large.
   // Returns true if the given area is zero filled upon completion.
-  bool commit(size_t start_page, size_t size_in_pages);
+  virtual bool commit(size_t start_page, size_t size_in_pages, uint node_index);
 
   // Uncommit the given area of pages starting at start being size_in_pages large.
-  void uncommit(size_t start_page, size_t size_in_pages);
+  virtual void uncommit(size_t start_page, size_t size_in_pages);
 
-  void pretouch(size_t start_page, size_t size_in_pages, WorkGang* pretouch_gang = NULL);
+  virtual void pretouch(size_t start_page, size_t size_in_pages, WorkGang* pretouch_gang = NULL);
 
   // Initialize the given reserved space with the given base address and the size
   // actually used.
   // Prefer to commit in page_size chunks.
-  G1PageBasedVirtualSpace(ReservedSpace rs, size_t used_size, size_t page_size);
+  G1PageBasedVirtualSpace(ReservedSpace rs, size_t used_size, size_t page_size, MemoryType type);
 
   // Destruction
   ~G1PageBasedVirtualSpace();
--- old/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp	2019-09-21 06:25:11.273962076 -0700
+++ new/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp	2019-09-21 06:25:10.909962089 -0700
@@ -24,6 +24,7 @@
 
 #include "precompiled.hpp"
 #include "gc/g1/g1BiasedArray.hpp"
+#include "gc/g1/g1MemoryNodeManager.hpp"
 #include "gc/g1/g1RegionToSpaceMapper.hpp"
 #include "logging/log.hpp"
 #include "memory/allocation.inline.hpp"
@@ -42,7 +43,7 @@
                                              size_t commit_factor,
                                              MemoryType type) :
   _listener(NULL),
-  _storage(rs, used_size, page_size),
+  _storage(rs, used_size, page_size, type),
   _region_granularity(region_granularity),
   _commit_map(rs.size() * commit_factor / region_granularity, mtGC) {
   guarantee(is_power_of_2(page_size), "must be");
@@ -71,11 +72,12 @@
     guarantee(alloc_granularity >= page_size, "allocation granularity smaller than commit granularity");
   }
 
-  virtual void commit_regions(uint start_idx, size_t num_regions, WorkGang* pretouch_gang) {
-    size_t const start_page = (size_t)start_idx * _pages_per_region;
-    bool zero_filled = _storage.commit(start_page, num_regions * _pages_per_region);
+  virtual void commit_regions(uint start_idx, size_t num_regions, uint node_index, WorkGang* pretouch_gang) {
+    const size_t start_page = (size_t)start_idx * _pages_per_region;
+    const size_t size_in_pages = num_regions * _pages_per_region;
+    bool zero_filled = _storage.commit(start_page, size_in_pages, node_index);
     if (AlwaysPreTouch) {
-      _storage.pretouch(start_page, num_regions * _pages_per_region, pretouch_gang);
+      _storage.pretouch(start_page, size_in_pages, pretouch_gang);
     }
     _commit_map.set_range(start_idx, start_idx + num_regions);
     fire_on_commit(start_idx, num_regions, zero_filled);
@@ -91,6 +93,36 @@
 // than the commit granularity.
 // Basically, the contents of one OS page span several regions.
 class G1RegionsSmallerThanCommitSizeMapper : public G1RegionToSpaceMapper {
+  // Helper class used to get node index evenly starting from the given node index.
+  // When the G1HeapRegionSize is smaller than page size, G1RegionToSpaceMapper will
+  // commit one heap region each time if multiple pages needs to be committed. In such case,
+  // node index also should reflect it. I.e. node indices should be used evenly.
+  class G1NodeDistributor : public StackObj {
+    uint _requested_node_index;
+    uint _next_node_index;
+    uint _max_node_index;
+  public:
+    G1NodeDistributor(uint node_index) :
+        _requested_node_index(node_index), 
+        // At the constructor body, _next_node_index will start from the first node index.
+        _next_node_index(G1MemoryNodeManager::mgr()->num_active_nodes() - 1),
+        _max_node_index(G1MemoryNodeManager::mgr()->num_active_nodes()) {
+      next();
+    }
+
+    uint next_node_index() const {
+      return _next_node_index;
+    }
+
+    void next() {
+      if (_requested_node_index == G1MemoryNodeManager::AnyNodeIndex) {
+        _next_node_index = (_next_node_index + 1) % _max_node_index;
+      } else {
+        _next_node_index = _requested_node_index;
+      }
+    }
+  };
+
  private:
   class CommitRefcountArray : public G1BiasedMappedArray<uint> {
    protected:
@@ -119,7 +151,7 @@
     _refcounts.initialize((HeapWord*)rs.base(), (HeapWord*)(rs.base() + align_up(rs.size(), page_size)), page_size);
   }
 
-  virtual void commit_regions(uint start_idx, size_t num_regions, WorkGang* pretouch_gang) {
+  virtual void commit_regions(uint start_idx, size_t num_regions, uint node_index, WorkGang* pretouch_gang) {
     size_t const NoPage = ~(size_t)0;
 
     size_t first_committed = NoPage;
@@ -127,7 +159,11 @@
 
     bool all_zero_filled = true;
 
+    G1NodeDistributor itr(node_index);
+
     for (uint i = start_idx; i < start_idx + num_regions; i++) {
+      // If there are many pages to touch, different node ids will be used.
+      uint processed_node_index = itr.next_node_index();
       assert(!_commit_map.at(i), "Trying to commit storage at region %u that is already committed", i);
       size_t idx = region_idx_to_page_idx(i);
       uint old_refcount = _refcounts.get_by_index(idx);
@@ -140,7 +176,8 @@
         } else {
           num_committed++;
         }
-        zero_filled = _storage.commit(idx, 1);
+        zero_filled = _storage.commit(idx, 1, processed_node_index);
+        itr.next();
       }
       all_zero_filled &= zero_filled;
 
@@ -254,7 +291,7 @@
   return true;
 }
 
-void G1RegionToHeteroSpaceMapper::commit_regions(uint start_idx, size_t num_regions, WorkGang* pretouch_gang) {
+void G1RegionToHeteroSpaceMapper::commit_regions(uint start_idx, size_t num_regions, uint node_index, WorkGang* pretouch_gang) {
   uint end_idx = (start_idx + (uint)num_regions - 1);
 
   uint num_dram = end_idx >= _start_index_of_dram ? MIN2((end_idx - _start_index_of_dram + 1), (uint)num_regions) : 0;
@@ -265,7 +302,7 @@
     _num_committed_nvdimm += num_nvdimm;
   }
   if (num_dram > 0) {
-    _dram_mapper->commit_regions(start_idx > _start_index_of_dram ? (start_idx - _start_index_of_dram) : 0, num_dram, pretouch_gang);
+    _dram_mapper->commit_regions(start_idx > _start_index_of_dram ? (start_idx - _start_index_of_dram) : 0, node_index, num_dram, pretouch_gang);
     _num_committed_dram += num_dram;
   }
 }
--- old/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp	2019-09-21 06:25:12.325962040 -0700
+++ new/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp	2019-09-21 06:25:11.941962053 -0700
@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_G1_G1REGIONTOSPACEMAPPER_HPP
 #define SHARE_GC_G1_G1REGIONTOSPACEMAPPER_HPP
 
+#include "gc/g1/g1MemoryNodeManager.hpp"
 #include "gc/g1/g1PageBasedVirtualSpace.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/debug.hpp"
@@ -71,7 +72,10 @@
   }
 
   void commit_and_set_special();
-  virtual void commit_regions(uint start_idx, size_t num_regions = 1, WorkGang* pretouch_workers = NULL) = 0;
+  virtual void commit_regions(uint start_idx,
+                              size_t num_regions = 1,
+                              uint node_index = G1MemoryNodeManager::AnyNodeIndex,
+                              WorkGang* pretouch_workers = NULL) = 0;
   virtual void uncommit_regions(uint start_idx, size_t num_regions = 1) = 0;
 
   // Creates an appropriate G1RegionToSpaceMapper for the given parameters.
@@ -116,7 +120,10 @@
   uint num_committed_dram() const;
   uint num_committed_nvdimm() const;
 
-  virtual void commit_regions(uint start_idx, size_t num_regions = 1, WorkGang* pretouch_workers = NULL);
+  virtual void commit_regions(uint start_idx,
+                              size_t num_regions = 1,
+                              uint node_index = G1MemoryNodeManager::AnyNodeIndex,
+                              WorkGang* pretouch_workers = NULL);
   virtual void uncommit_regions(uint start_idx, size_t num_regions = 1);
 };
 #endif // SHARE_GC_G1_G1REGIONTOSPACEMAPPER_HPP
--- old/src/hotspot/share/gc/g1/g1_globals.hpp	2019-09-21 06:25:13.417962002 -0700
+++ new/src/hotspot/share/gc/g1/g1_globals.hpp	2019-09-21 06:25:13.057962014 -0700
@@ -324,6 +324,13 @@
                "absorb small changes in young gen length. This flag takes " \
                "the buffer size as an percentage of young gen length")      \
                range(0, 100)                                                \
+                                                                            \
+  diagnostic(bool, G1PrintNUMAIdOfHeapRegions, false,                       \
+             "Print NUMA id of committed HeapRegion(s)")                    \
+                                                                            \
+  diagnostic(bool, G1VerifyNUMAIdOfHeapRegions, false,                      \
+             "Verify NUMA id of committed HeapRegion(s) whether those "     \
+             "regions are still on the firstly checked node or not.")
 
 
 #endif // SHARE_GC_G1_G1_GLOBALS_HPP
--- old/src/hotspot/share/gc/g1/heapRegion.cpp	2019-09-21 06:25:14.485961964 -0700
+++ new/src/hotspot/share/gc/g1/heapRegion.cpp	2019-09-21 06:25:14.121961977 -0700
@@ -28,6 +28,7 @@
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectionSet.hpp"
 #include "gc/g1/g1HeapRegionTraceType.hpp"
+#include "gc/g1/g1MemoryNodeManager.hpp"
 #include "gc/g1/g1OopClosures.inline.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionBounds.inline.hpp"
@@ -248,7 +249,8 @@
     _index_in_opt_cset(InvalidCSetIndex), _young_index_in_cset(-1),
     _surv_rate_group(NULL), _age_index(-1),
     _prev_top_at_mark_start(NULL), _next_top_at_mark_start(NULL),
-    _recorded_rs_length(0), _predicted_elapsed_time_ms(0)
+    _recorded_rs_length(0), _predicted_elapsed_time_ms(0),
+    _node_index(G1MemoryNodeManager::InvalidNodeIndex)
 {
   _rem_set = new HeapRegionRemSet(bot, this);
 
--- old/src/hotspot/share/gc/g1/heapRegion.hpp	2019-09-21 06:25:15.585961926 -0700
+++ new/src/hotspot/share/gc/g1/heapRegion.hpp	2019-09-21 06:25:15.221961939 -0700
@@ -291,6 +291,8 @@
   // for the collection set.
   double _predicted_elapsed_time_ms;
 
+  uint _node_index;
+
   // Iterate over the references covered by the given MemRegion in a humongous
   // object and apply the given closure to them.
   // Humongous objects are allocated directly in the old-gen. So we need special
@@ -685,6 +687,9 @@
   // the strong code roots list for this region
   void strong_code_roots_do(CodeBlobClosure* blk) const;
 
+  uint node_index() const { return _node_index; }
+  void set_node_index(uint node_index) { _node_index = node_index; }
+
   // Verify that the entries on the strong code root list for this
   // region are live and include at least one pointer into this region.
   void verify_strong_code_roots(VerifyOption vo, bool* failures) const;
--- old/src/hotspot/share/gc/g1/heapRegionManager.cpp	2019-09-21 06:25:16.717961887 -0700
+++ new/src/hotspot/share/gc/g1/heapRegionManager.cpp	2019-09-21 06:25:16.357961899 -0700
@@ -30,6 +30,7 @@
 #include "gc/g1/heapRegionManager.inline.hpp"
 #include "gc/g1/heapRegionSet.inline.hpp"
 #include "gc/g1/heterogeneousHeapRegionManager.hpp"
+#include "logging/logStream.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/bitMap.inline.hpp"
 
@@ -103,6 +104,40 @@
   return _available_map.at(region);
 }
 
+HeapRegion* HeapRegionManager::allocate_free_region(HeapRegionType type, uint requested_node_index) {
+  G1MemoryNodeManager* mgr = G1MemoryNodeManager::mgr();
+  HeapRegion* hr = NULL;
+  bool from_head = !type.is_young();
+
+  if (mgr->num_active_nodes() > 1) {
+    uint valid_node_index = mgr->valid_node_index(requested_node_index);
+    // Try to allocate with requested node index.
+    hr = _free_list.remove_region_with_node_index(from_head, valid_node_index, NULL);
+  }
+
+  if (hr == NULL) {
+    // If there's a single active node or we did not get a region from our requested node,
+    // try without requested node index.
+    hr = _free_list.remove_region(from_head);
+  }
+
+  if (hr != NULL) {
+    assert(hr->next() == NULL, "Single region should not have next");
+    assert(is_available(hr->hrm_index()), "Must be committed");
+    if (G1VerifyNUMAIdOfHeapRegions) {
+      // Read actual node index via system call.
+      uint actual_node_index = mgr->index_of_address(hr->bottom());
+      if (hr->node_index() != actual_node_index) {
+        log_debug(gc, heap, numa)("Heap Region (%u) has different node index. "
+                                  "actual index=%u, index=%u",
+                                  hr->hrm_index(), actual_node_index, hr->node_index());
+      }
+    }
+  }
+
+  return hr;
+}
+
 #ifdef ASSERT
 bool HeapRegionManager::is_free(HeapRegion* hr) const {
   return _free_list.contains(hr);
@@ -117,28 +152,33 @@
   return g1h->new_heap_region(hrm_index, mr);
 }
 
-void HeapRegionManager::commit_regions(uint index, size_t num_regions, WorkGang* pretouch_gang) {
+void HeapRegionManager::commit_regions(uint index, size_t num_regions, uint node_index, WorkGang* pretouch_gang) {
   guarantee(num_regions > 0, "Must commit more than zero regions");
   guarantee(_num_committed + num_regions <= max_length(), "Cannot commit more than the maximum amount of regions");
 
   _num_committed += (uint)num_regions;
 
-  _heap_mapper->commit_regions(index, num_regions, pretouch_gang);
+  _heap_mapper->commit_regions(index, num_regions, node_index, pretouch_gang);
 
   // Also commit auxiliary data
-  _prev_bitmap_mapper->commit_regions(index, num_regions, pretouch_gang);
-  _next_bitmap_mapper->commit_regions(index, num_regions, pretouch_gang);
+  _prev_bitmap_mapper->commit_regions(index, num_regions, node_index, pretouch_gang);
+  _next_bitmap_mapper->commit_regions(index, num_regions, node_index, pretouch_gang);
 
-  _bot_mapper->commit_regions(index, num_regions, pretouch_gang);
-  _cardtable_mapper->commit_regions(index, num_regions, pretouch_gang);
+  _bot_mapper->commit_regions(index, num_regions, node_index, pretouch_gang);
+  _cardtable_mapper->commit_regions(index, num_regions, node_index, pretouch_gang);
 
-  _card_counts_mapper->commit_regions(index, num_regions, pretouch_gang);
+  _card_counts_mapper->commit_regions(index, num_regions, node_index, pretouch_gang);
 }
 
 void HeapRegionManager::uncommit_regions(uint start, size_t num_regions) {
   guarantee(num_regions >= 1, "Need to specify at least one region to uncommit, tried to uncommit zero regions at %u", start);
   guarantee(_num_committed >= num_regions, "pre-condition");
 
+  // Reset node index to distinguish with committed regions.
+  for (uint i = start; i < start + num_regions; i++) {
+    at(i)->set_node_index(G1MemoryNodeManager::InvalidNodeIndex);
+  }
+
   // Print before uncommitting.
   if (G1CollectedHeap::heap()->hr_printer()->is_active()) {
     for (uint i = start; i < start + num_regions; i++) {
@@ -162,9 +202,26 @@
   _card_counts_mapper->uncommit_regions(start, num_regions);
 }
 
-void HeapRegionManager::make_regions_available(uint start, uint num_regions, WorkGang* pretouch_gang) {
+static void print_numa_id_of_regions(uint start, uint num_regions) {
+  LogTarget(Debug, gc, heap, numa) lt;
+
+  if (lt.is_enabled()) {
+    LogStream ls(lt);
+
+    // Print header
+    // Below logs are checked by TestG1NUMATouchRegions.java.
+    ls.print_cr("Numa id of heap regions from %u to %u", start, start + num_regions - 1);
+    ls.print_cr("Heap Region# : numa id of pages");
+
+    for (uint i = start; i < start + num_regions; i++) {
+      ls.print_cr("%6u : %u", i, G1CollectedHeap::heap()->region_at(i)->node_index());
+    }
+  }
+}
+
+void HeapRegionManager::make_regions_available(uint start, uint num_regions, uint node_index, WorkGang* pretouch_gang) {
   guarantee(num_regions > 0, "No point in calling this for zero regions");
-  commit_regions(start, num_regions, pretouch_gang);
+  commit_regions(start, num_regions, node_index, pretouch_gang);
   for (uint i = start; i < start + num_regions; i++) {
     if (_regions.get_by_index(i) == NULL) {
       HeapRegion* new_hr = new_heap_region(i);
@@ -187,6 +244,12 @@
 
     hr->initialize(mr);
     insert_into_free_list(at(i));
+    // Set node index of the heap region after initialization.
+    hr->set_node_index(G1MemoryNodeManager::mgr()->index_of_address(bottom));
+  }
+
+  if (G1PrintNUMAIdOfHeapRegions) {
+    print_numa_id_of_regions(start, num_regions);
   }
 }
 
@@ -208,11 +271,11 @@
   return MemoryUsage(0, used_sz, committed_sz, committed_sz);
 }
 
-uint HeapRegionManager::expand_by(uint num_regions, WorkGang* pretouch_workers) {
-  return expand_at(0, num_regions, pretouch_workers);
+uint HeapRegionManager::expand_by(uint num_regions, uint node_index, WorkGang* pretouch_workers) {
+  return expand_at(0, num_regions, node_index, pretouch_workers);
 }
 
-uint HeapRegionManager::expand_at(uint start, uint num_regions, WorkGang* pretouch_workers) {
+uint HeapRegionManager::expand_at(uint start, uint num_regions, uint node_index, WorkGang* pretouch_workers) {
   if (num_regions == 0) {
     return 0;
   }
@@ -226,7 +289,7 @@
   while (expanded < num_regions &&
          (num_last_found = find_unavailable_from_idx(cur, &idx_last_found)) > 0) {
     uint to_expand = MIN2(num_regions - expanded, num_last_found);
-    make_regions_available(idx_last_found, to_expand, pretouch_workers);
+    make_regions_available(idx_last_found, to_expand, node_index, pretouch_workers);
     expanded += to_expand;
     cur = idx_last_found + num_last_found + 1;
   }
@@ -331,7 +394,7 @@
   while (true) {
     HeapRegion *hr = _regions.get_by_index(curr);
     if (hr == NULL || !is_available(curr)) {
-      uint res = expand_at(curr, 1, NULL);
+      uint res = expand_at(curr, 1, G1MemoryNodeManager::AnyNodeIndex, NULL);
       if (res == 1) {
         *expanded = true;
         return curr;
@@ -359,7 +422,7 @@
   for (uint curr_index = start_index; curr_index <= last_index; curr_index++) {
     if (!is_available(curr_index)) {
       commits++;
-      expand_at(curr_index, 1, pretouch_workers);
+      expand_at(curr_index, 1, G1MemoryNodeManager::AnyNodeIndex, pretouch_workers);
     }
     HeapRegion* curr_region  = _regions.get_by_index(curr_index);
     if (!curr_region->is_free()) {
--- old/src/hotspot/share/gc/g1/heapRegionManager.hpp	2019-09-21 06:25:17.773961850 -0700
+++ new/src/hotspot/share/gc/g1/heapRegionManager.hpp	2019-09-21 06:25:17.405961863 -0700
@@ -89,7 +89,10 @@
   HeapWord* heap_end() const {return _regions.end_address_mapped(); }
 
   // Pass down commit calls to the VirtualSpace.
-  void commit_regions(uint index, size_t num_regions = 1, WorkGang* pretouch_gang = NULL);
+  void commit_regions(uint index,
+                      size_t num_regions = 1,
+                      uint node_index = G1MemoryNodeManager::AnyNodeIndex,
+                      WorkGang* pretouch_gang = NULL);
 
   // Notify other data structures about change in the heap layout.
   void update_committed_space(HeapWord* old_end, HeapWord* new_end);
@@ -115,7 +118,10 @@
   G1RegionToSpaceMapper* _next_bitmap_mapper;
   FreeRegionList _free_list;
 
-  void make_regions_available(uint index, uint num_regions = 1, WorkGang* pretouch_gang = NULL);
+  void make_regions_available(uint index,
+                              uint num_regions = 1,
+                              uint node_index = G1MemoryNodeManager::AnyNodeIndex,
+                              WorkGang* pretouch_gang = NULL);
   void uncommit_regions(uint index, size_t num_regions = 1);
   // Allocate a new HeapRegion for the given index.
   HeapRegion* new_heap_region(uint hrm_index);
@@ -174,15 +180,8 @@
     _free_list.add_ordered(list);
   }
 
-  virtual HeapRegion* allocate_free_region(HeapRegionType type) {
-    HeapRegion* hr = _free_list.remove_region(!type.is_young());
-
-    if (hr != NULL) {
-      assert(hr->next() == NULL, "Single region should not have next");
-      assert(is_available(hr->hrm_index()), "Must be committed");
-    }
-    return hr;
-  }
+  // Allocate a free region with specific node index. If fails allocate with next node index.
+  virtual HeapRegion* allocate_free_region(HeapRegionType type, uint requested_node_index);
 
   inline void allocate_free_regions_starting_at(uint first, uint num_regions);
 
@@ -220,12 +219,12 @@
   // HeapRegions, or re-use existing ones. Returns the number of regions the
   // sequence was expanded by. If a HeapRegion allocation fails, the resulting
   // number of regions might be smaller than what's desired.
-  virtual uint expand_by(uint num_regions, WorkGang* pretouch_workers);
+  virtual uint expand_by(uint num_regions, uint node_index, WorkGang* pretouch_workers);
 
   // Makes sure that the regions from start to start+num_regions-1 are available
   // for allocation. Returns the number of regions that were committed to achieve
   // this.
-  virtual uint expand_at(uint start, uint num_regions, WorkGang* pretouch_workers);
+  virtual uint expand_at(uint start, uint num_regions, uint node_index, WorkGang* pretouch_workers);
 
   // Find a contiguous set of empty regions of length num. Returns the start index of
   // that set, or G1_NO_HRM_INDEX.
--- old/src/hotspot/share/gc/g1/heapRegionSet.hpp	2019-09-21 06:25:18.889961811 -0700
+++ new/src/hotspot/share/gc/g1/heapRegionSet.hpp	2019-09-21 06:25:18.525961824 -0700
@@ -181,6 +181,10 @@
   // Removes from head or tail based on the given argument.
   HeapRegion* remove_region(bool from_head);
 
+  HeapRegion* remove_region_with_node_index(bool from_head,
+                                            const uint requested_node_index,
+                                            uint* region_node_index);
+
   // Merge two ordered lists. The result is also ordered. The order is
   // determined by hrm_index.
   void add_ordered(FreeRegionList* from_list);
--- old/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp	2019-09-21 06:25:19.993961773 -0700
+++ new/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp	2019-09-21 06:25:19.625961785 -0700
@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
 #define SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
 
+#include "gc/g1/g1NUMA.inline.hpp"
 #include "gc/g1/heapRegionSet.hpp"
 
 inline void HeapRegionSetBase::add(HeapRegion* hr) {
@@ -147,4 +148,72 @@
   return hr;
 }
 
+inline HeapRegion* FreeRegionList::remove_region_with_node_index(bool from_head,
+                                                                 const uint requested_node_index,
+                                                                 uint* allocated_node_index) {
+  assert(UseNUMA, "Invariant");
+
+  HeapRegion * cur;
+  G1NUMA* numa = G1NUMA::numa();
+
+  if (!numa->is_valid_numa_index(requested_node_index)) {
+    return NULL;
+  }
+
+  // Multiple of 3 is just random number to limit iterations.
+  uint const max_search_depth = 3 * numa->num_active_numa_ids();
+
+  // Find the region to use, searching from _head or _tail as requested.
+  size_t cur_depth = 0;
+  if (from_head) {
+    for (cur = _head;
+         cur != NULL && cur_depth < max_search_depth;
+         cur = cur->next(), ++cur_depth) {
+      if (requested_node_index == cur->node_index()) {
+        break;
+      }
+    }
+  } else {
+    for (cur = _tail;
+         cur != NULL && cur_depth < max_search_depth;
+         cur = cur->prev(), ++cur_depth) {
+      if (requested_node_index == cur->node_index()) {
+        break;
+      }
+    }
+  }
+
+  // Didn't find a region to use.
+  if (cur == NULL || cur_depth >= max_search_depth) {
+    return NULL;
+  }
+
+  // Splice the region out of the list.
+  HeapRegion* prev = cur->prev();
+  HeapRegion* next = cur->next();
+  if (prev == NULL) {
+    _head = next;
+  } else {
+    prev->set_next(next);
+  }
+  if (next == NULL) {
+    _tail = prev;
+  } else {
+    next->set_prev(prev);
+  }
+  cur->set_prev(NULL);
+  cur->set_next(NULL);
+
+  if (_last == cur) {
+    _last = NULL;
+  }
+
+  remove(cur);
+  if (allocated_node_index != NULL) {
+    *allocated_node_index = cur->node_index();
+  }
+
+  return cur;
+}
+
 #endif // SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
--- old/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp	2019-09-21 06:25:21.025961737 -0700
+++ new/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp	2019-09-21 06:25:20.661961749 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -58,7 +58,7 @@
 // expand_by() is called to grow the heap. We grow into nvdimm now.
 // Dram regions are committed later as needed during mutator region allocation or
 // when young list target length is determined after gc cycle.
-uint HeterogeneousHeapRegionManager::expand_by(uint num_regions, WorkGang* pretouch_workers) {
+uint HeterogeneousHeapRegionManager::expand_by(uint num_regions, uint node_index, WorkGang* pretouch_workers) {
   uint num_regions_possible = total_regions_committed() >= max_expandable_length() ? 0 : max_expandable_length() - total_regions_committed();
   uint num_expanded = expand_nvdimm(MIN2(num_regions, num_regions_possible), pretouch_workers);
   return num_expanded;
@@ -67,7 +67,7 @@
 // Expands heap starting from 'start' index. The question is should we expand from one memory (e.g. nvdimm) to another (e.g. dram).
 // Looking at the code, expand_at() is called for humongous allocation where 'start' is in nv-dimm.
 // So we only allocate regions in the same kind of memory as 'start'.
-uint HeterogeneousHeapRegionManager::expand_at(uint start, uint num_regions, WorkGang* pretouch_workers) {
+uint HeterogeneousHeapRegionManager::expand_at(uint start, uint num_regions, uint node_index, WorkGang* pretouch_workers) {
   if (num_regions == 0) {
     return 0;
   }
@@ -190,7 +190,7 @@
   while (so_far < num_regions &&
          (num_last_found = find_unavailable_in_range(start, end, &chunk_start)) > 0) {
     uint to_commit = MIN2(num_regions - so_far, num_last_found);
-    make_regions_available(chunk_start, to_commit, pretouch_gang);
+    make_regions_available(chunk_start, to_commit, G1MemoryNodeManager::AnyNodeIndex, pretouch_gang);
     so_far += to_commit;
     start = chunk_start + to_commit + 1;
   }
@@ -263,7 +263,7 @@
   return num_regions_found;
 }
 
-HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(HeapRegionType type) {
+HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(HeapRegionType type, uint node_index) {
 
   // We want to prevent mutators from proceeding when we have borrowed regions from the last collection. This
   // will force a full collection to remedy the situation.
--- old/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp	2019-09-21 06:25:22.113961699 -0700
+++ new/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp	2019-09-21 06:25:21.745961712 -0700
@@ -119,16 +119,16 @@
   void prepare_for_full_collection_start();
   void prepare_for_full_collection_end();
 
-  virtual HeapRegion* allocate_free_region(HeapRegionType type);
+  virtual HeapRegion* allocate_free_region(HeapRegionType type, uint node_index);
 
   // Return maximum number of regions that heap can expand to.
   uint max_expandable_length() const;
 
   // Override. Expand in nv-dimm.
-  uint expand_by(uint num_regions, WorkGang* pretouch_workers);
+  uint expand_by(uint num_regions, uint node_index, WorkGang* pretouch_workers);
 
   // Override.
-  uint expand_at(uint start, uint num_regions, WorkGang* pretouch_workers);
+  uint expand_at(uint start, uint num_regions, uint node_index, WorkGang* pretouch_workers);
 
   // Override. This function is called for humongous allocation, so we need to find empty regions in nv-dimm.
   uint find_contiguous_only_empty(size_t num);
--- old/src/hotspot/share/logging/logPrefix.hpp	2019-09-21 06:25:25.065961596 -0700
+++ new/src/hotspot/share/logging/logPrefix.hpp	2019-09-21 06:25:24.737961607 -0700
@@ -57,6 +57,7 @@
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ergo, ihop)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ergo, refine)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap)) \
+  LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap, numa)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap, region)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, freelist)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, humongous)) \
--- old/src/hotspot/share/logging/logTag.hpp	2019-09-21 06:25:26.105961560 -0700
+++ new/src/hotspot/share/logging/logTag.hpp	2019-09-21 06:25:25.749961572 -0700
@@ -108,6 +108,7 @@
   LOG_TAG(nestmates) \
   LOG_TAG(nmethod) \
   LOG_TAG(normalize) \
+  LOG_TAG(numa) \
   LOG_TAG(objecttagging) \
   LOG_TAG(obsolete) \
   LOG_TAG(oldobject) \
--- old/src/hotspot/share/prims/whitebox.cpp	2019-09-21 06:25:27.193961522 -0700
+++ new/src/hotspot/share/prims/whitebox.cpp	2019-09-21 06:25:26.829961534 -0700
@@ -619,6 +619,29 @@
   THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1AuxiliaryMemoryUsage: G1 GC is not enabled");
 WB_END
 
+WB_ENTRY(jint, WB_G1ActiveMemoryNodeCount(JNIEnv* env, jobject o))
+  if (UseG1GC) {
+    G1MemoryNodeManager* mgr = G1MemoryNodeManager::mgr();
+    return (jint)mgr->num_active_nodes();
+  }
+  THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1ActiveMemoryNodeCount: G1 GC is not enabled");
+WB_END
+
+WB_ENTRY(jintArray, WB_G1MemoryNodeIds(JNIEnv* env, jobject o))
+  if (UseG1GC) {
+    G1MemoryNodeManager* mgr = G1MemoryNodeManager::mgr();
+    int num_node_ids = (int)mgr->num_active_nodes();
+    const int* node_ids = mgr->node_ids();
+
+    typeArrayOop result = oopFactory::new_intArray(num_node_ids, CHECK_NULL);
+    for (int i = 0; i < num_node_ids; i++) {
+      result->int_at_put(i, (jint)node_ids[i]);
+    }
+    return (jintArray) JNIHandles::make_local(env, result);
+  }
+  THROW_MSG_NULL(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1MemoryNodeIds: G1 GC is not enabled");
+WB_END
+
 class OldRegionsLivenessClosure: public HeapRegionClosure {
 
  private:
@@ -2185,6 +2208,8 @@
   {CC"g1StartConcMarkCycle",       CC"()Z",           (void*)&WB_G1StartMarkCycle  },
   {CC"g1AuxiliaryMemoryUsage", CC"()Ljava/lang/management/MemoryUsage;",
                                                       (void*)&WB_G1AuxiliaryMemoryUsage  },
+  {CC"g1ActiveMemoryNodeCount", CC"()I",              (void*)&WB_G1ActiveMemoryNodeCount },
+  {CC"g1MemoryNodeIds",    CC"()[I",                  (void*)&WB_G1MemoryNodeIds },
   {CC"g1GetMixedGCInfo",   CC"(I)[J",                 (void*)&WB_G1GetMixedGCInfo },
 #endif // INCLUDE_G1GC
 #if INCLUDE_G1GC || INCLUDE_PARALLELGC
--- old/src/hotspot/share/runtime/arguments.cpp	2019-09-21 06:25:28.277961484 -0700
+++ new/src/hotspot/share/runtime/arguments.cpp	2019-09-21 06:25:27.917961497 -0700
@@ -4113,10 +4113,10 @@
     }
     // UseNUMAInterleaving is set to ON for all collectors and
     // platforms when UseNUMA is set to ON. NUMA-aware collectors
-    // such as the parallel collector for Linux and Solaris will
+    // such as Parallel GC for Linux and Solaris or G1 GC for Linux will
     // interleave old gen and survivor spaces on top of NUMA
     // allocation policy for the eden space.
-    // Non NUMA-aware collectors such as CMS, G1 and Serial-GC on
+    // Non NUMA-aware collectors such as CMS and Serial-GC on
     // all platforms and ParallelGC on Windows will interleave all
     // of the heap spaces across NUMA nodes.
     if (FLAG_IS_DEFAULT(UseNUMAInterleaving)) {
--- old/src/hotspot/share/runtime/os.hpp	2019-09-21 06:25:29.401961445 -0700
+++ new/src/hotspot/share/runtime/os.hpp	2019-09-21 06:25:29.037961458 -0700
@@ -390,6 +390,13 @@
   static bool   numa_topology_changed();
   static int    numa_get_group_id();
 
+  enum NumaIdState {
+    InvalidId = -1,
+    AnyId = -2
+  };
+
+  static int numa_get_address_id(uintptr_t address);
+
   // Page manipulation
   struct page_info {
     size_t size;
--- old/test/hotspot/gtest/gc/g1/test_freeRegionList.cpp	2019-09-21 06:25:30.509961406 -0700
+++ new/test/hotspot/gtest/gc/g1/test_freeRegionList.cpp	2019-09-21 06:25:30.153961419 -0700
@@ -51,7 +51,7 @@
                                          BOTConstants::N_bytes,
                                          mtGC);
   G1BlockOffsetTable bot(heap, bot_storage);
-  bot_storage->commit_regions(0, num_regions_in_test);
+  bot_storage->commit_regions(0, num_regions_in_test, G1MemoryNodeManager::AnyNodeIndex);
 
   // Set up memory regions for the heap regions.
   MemRegion mr0(heap.start(), HeapRegion::GrainWords);
--- old/test/lib/sun/hotspot/WhiteBox.java	2019-09-21 06:25:31.605961368 -0700
+++ new/test/lib/sun/hotspot/WhiteBox.java	2019-09-21 06:25:31.237961381 -0700
@@ -193,6 +193,9 @@
     return parseCommandLine0(commandline, delim, args);
   }
 
+  public native int g1ActiveMemoryNodeCount();
+  public native int[] g1MemoryNodeIds();
+
   // Parallel GC
   public native long psVirtualSpaceAlignment();
   public native long psHeapGenerationAlignment();
--- /dev/null	2019-09-04 10:58:08.784555846 -0700
+++ new/src/hotspot/share/gc/g1/g1MemoryNodeManager.cpp	2019-09-21 06:25:32.357961342 -0700
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/heapRegionType.hpp"
+#include "gc/g1/g1MemoryNodeManager.hpp"
+#include "gc/g1/g1NUMA.inline.hpp"
+
+G1MemoryNodeManager* G1MemoryNodeManager::_inst = NULL;
+
+class G1MemoryMultiNodeManager : public G1MemoryNodeManager {
+  G1NUMA*      _numa;
+
+public:
+  // The given numa instance will be deleted by the destructor.
+  G1MemoryMultiNodeManager(G1NUMA* numa);
+  ~G1MemoryMultiNodeManager();
+
+  virtual void set_page_size(size_t page_size);
+
+  virtual uint num_active_nodes() const;
+
+  virtual const int* node_ids() const;
+
+  virtual uint index_of_current_thread() const;
+
+  virtual uint valid_node_index(uint node_index) const;
+
+  virtual bool is_valid_node_index(uint node_index) const;
+
+  virtual uint index_of_address(HeapWord* addr) const;
+};
+
+G1MemoryNodeManager* G1MemoryNodeManager::create() {
+  guarantee(_inst == NULL, "Should be called once.");
+
+  // Use multi node manager:
+  //   1. If UseNUMA is enabled
+  //   2. If there are more than 1 active numa nodes.
+  // Doesn't support multi node manager for Solaris.
+  if (UseNUMA SOLARIS_ONLY(&& false)) {
+    // Create G1NUMA to check current active numa nodes.
+    G1NUMA* numa = new G1NUMA();
+
+    if (numa != NULL) {
+      numa->initialize();
+
+      if (numa->num_active_numa_ids() > 1) {
+        G1NUMA::set_numa(numa);
+        _inst = new G1MemoryMultiNodeManager(numa);
+        return _inst;
+      }
+
+      delete numa;
+    }
+  }
+
+  _inst = new G1MemoryNodeManager();
+
+  return _inst;
+}
+
+G1MemoryMultiNodeManager::G1MemoryMultiNodeManager(G1NUMA* numa) : _numa(numa) { }
+
+G1MemoryMultiNodeManager::~G1MemoryMultiNodeManager() {
+  delete _numa;
+}
+
+void G1MemoryMultiNodeManager::set_page_size(size_t page_size) {
+  _numa->set_page_size(page_size);
+}
+
+uint G1MemoryMultiNodeManager::num_active_nodes() const {
+  return _numa->num_active_numa_ids();
+}
+
+const int* G1MemoryMultiNodeManager::node_ids() const {
+  return _numa->numa_ids();
+}
+
+uint G1MemoryMultiNodeManager::index_of_current_thread() const {
+  int node_id = os::numa_get_group_id();
+  return _numa->index_of_numa_id(node_id);
+}
+
+uint G1MemoryMultiNodeManager::valid_node_index(uint node_index) const {
+  if (!_numa->is_valid_numa_index(node_index)) {
+    node_index = _numa->next_numa_index();
+  }
+  return node_index;
+}
+
+bool G1MemoryMultiNodeManager::is_valid_node_index(uint node_index) const {
+  return _numa->is_valid_numa_index(node_index);
+}
+
+uint G1MemoryMultiNodeManager::index_of_address(HeapWord* addr) const {
+  return _numa->index_of_address(addr);
+}
--- /dev/null	2019-09-04 10:58:08.784555846 -0700
+++ new/src/hotspot/share/gc/g1/g1MemoryNodeManager.hpp	2019-09-21 06:25:33.421961305 -0700
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_G1_MEMORY_NODE_MANAGER_HPP
+#define SHARE_VM_GC_G1_MEMORY_NODE_MANAGER_HPP
+
+#include "memory/allocation.hpp"
+#include "runtime/os.hpp"
+
+// Helper class to manage memory nodes.
+// G1MemoryMultiNodeManager will be created if UseNUMA is enabled and active
+// NUMA nodes are more than one. Otherwise, G1MemoryNodeManager will be created.
+class G1MemoryNodeManager : public CHeapObj<mtGC> {
+  static G1MemoryNodeManager* _inst;
+
+public:
+  static const uint InvalidNodeIndex = (uint)os::InvalidId;
+  static const uint AnyNodeIndex = (uint)os::AnyId;
+
+  static G1MemoryNodeManager* mgr() { return _inst; }
+
+  static G1MemoryNodeManager* create();
+
+  virtual ~G1MemoryNodeManager() { }
+
+  // Set page size of Java Heap.
+  virtual void set_page_size(size_t page_size) { }
+
+  // Print current active memory node count.
+  virtual uint num_active_nodes() const { return 1; }
+
+  // Returns memory node ids
+  virtual const int* node_ids() const { static int dummy_id = 0; return &dummy_id; }
+
+  virtual uint index_of_current_thread() const { return 0; }
+
+  // If the given node index is valid return same index,
+  // If it is not valid, generate valid random index.
+  virtual uint valid_node_index(uint node_index) const { return 0; }
+
+  virtual bool is_valid_node_index(uint node_index) const {
+    // Single node index should be always 0.
+    return node_index == 0;
+  }
+
+  // Retrieve node index of the given address.
+  virtual uint index_of_address(HeapWord* addr) const { return 0; }
+};
+
+#endif // SHARE_VM_GC_G1_MEMORY_NODE_MANAGER_HPP
--- /dev/null	2019-09-04 10:58:08.784555846 -0700
+++ new/src/hotspot/share/gc/g1/g1NUMA.cpp	2019-09-21 06:25:34.541961266 -0700
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1NUMA.inline.hpp"
+#include "gc/g1/heapRegion.hpp"
+
+G1NUMA* G1NUMA::_inst = NULL;
+
+void G1NUMA::init_numa_id_to_index_map(const int* numa_ids, uint num_numa_ids) {
+  int max_numa_id = 0;
+  for (uint i = 0; i < num_numa_ids; i++) {
+    if (numa_ids[i] > max_numa_id) {
+      max_numa_id = numa_ids[i];
+    }
+  }
+
+  _len_numa_id_to_index_map = max_numa_id + 1;
+  _numa_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_numa_id_to_index_map, mtGC);
+  // Set all indices with invalid numa id.
+  memset(_numa_id_to_index_map, 
+         G1MemoryNodeManager::InvalidNodeIndex,
+         sizeof(uint) * _len_numa_id_to_index_map);
+
+  // Set the indices for the actually retrieved numa ids.
+  for (uint i = 0; i < num_numa_ids; i++) {
+     int numa_id = numa_ids[i];
+    guarantee(is_valid_numa_id(numa_id), "must be representable in map, numa id(%d)", numa_id);
+     _numa_id_to_index_map[numa_id] = i;
+  }
+}
+
+void G1NUMA::touch_memory_whole(address aligned_address, size_t size_in_bytes, uint numa_index) {
+    log_debug(gc, heap, numa)("Touch memory [" PTR_FORMAT ", " PTR_FORMAT ") to be numa id (%d).",
+                             p2i(aligned_address), p2i(aligned_address + size_in_bytes), _numa_ids[numa_index]);
+
+    // If we have preferred numa id, set the whole given area with it.
+    os::numa_make_local((char*)aligned_address, size_in_bytes, _numa_ids[numa_index]);
+}
+
+void G1NUMA::touch_memory_roundrobin(address aligned_address, size_t size_in_bytes) {
+  // If we don't have preferred numa id, touch the given area with round-robin manner.
+  size_t chunk_size;
+  if (HeapRegion::GrainBytes >= _page_size) {
+    chunk_size = HeapRegion::GrainBytes;
+  } else {
+    chunk_size = _page_size;
+  }
+
+  assert(is_aligned(size_in_bytes, chunk_size), "Size to touch " SIZE_FORMAT " should be aligned to " SIZE_FORMAT,
+         size_in_bytes, chunk_size);
+
+  address start_addr = aligned_address;
+  address end_addr = aligned_address + size_in_bytes;
+
+  log_debug(gc, heap, numa)("Start touch memory [" PTR_FORMAT ", " PTR_FORMAT
+                            "), chunk_size=" SIZE_FORMAT " with round-robin manner.",
+                            p2i(start_addr), p2i(end_addr), chunk_size);
+
+  do {
+    uint numa_index = next_numa_index();
+
+    log_trace(gc, heap, numa)("Touch memory [" PTR_FORMAT ", " PTR_FORMAT ") to be numa id (%d).",
+                             p2i(start_addr), p2i(start_addr + chunk_size), _numa_ids[numa_index]);
+    os::numa_make_local((char*)start_addr, chunk_size, _numa_ids[numa_index]);
+
+    start_addr += chunk_size;
+  } while (start_addr < end_addr);
+}
+
+void G1NUMA::touch_memory(address aligned_address, size_t size_in_bytes, uint numa_index) {
+  if (size_in_bytes == 0) {
+    return;
+  }
+
+  if (is_valid_numa_index(numa_index)) {
+    touch_memory_whole(aligned_address, size_in_bytes,  numa_index);
+  } else {
+    touch_memory_roundrobin(aligned_address, size_in_bytes);
+  }
+}
+
+bool G1NUMA::initialize() {
+  assert(UseNUMA, "Invariant");
+
+  size_t num_numa_ids = os::numa_get_groups_num();
+
+  _numa_ids = NEW_C_HEAP_ARRAY(int, num_numa_ids, mtGC);
+  _num_active_numa_ids = (uint)os::numa_get_leaf_groups(_numa_ids, num_numa_ids);
+
+  // To start from 0 at the first call of next_numa_index(), set to the greatest one.
+  _next_numa_index = _num_active_numa_ids - 1;
+
+  init_numa_id_to_index_map(_numa_ids, _num_active_numa_ids);
+
+  return true;
+}
+
+void G1NUMA::set_page_size(size_t page_size) {
+  _page_size = page_size;
+}
+
+G1NUMA::~G1NUMA() {
+  FREE_C_HEAP_ARRAY(int, _numa_id_to_index_map);
+  FREE_C_HEAP_ARRAY(int, _numa_ids);
+}
+
+uint G1NUMA::index_of_address(HeapWord* addr) const {
+  int numa_id = os::numa_get_address_id((uintptr_t)addr);
+
+  return index_of_numa_id(numa_id);
+}
--- /dev/null	2019-09-04 10:58:08.784555846 -0700
+++ new/src/hotspot/share/gc/g1/g1NUMA.hpp	2019-09-21 06:25:35.633961228 -0700
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_G1_G1NUMA_HPP
+#define SHARE_VM_GC_G1_G1NUMA_HPP
+
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+
+// Manages NUMA node related information.
+// Provides a conversion between NUMA indices (sequential, starting
+// from zero) and NUMA ids (which may not start at zero and may not be
+// sequential). Indices are generally used by clients, and mapped to
+// ids when dealing with the OS/hardware layer.
+class G1NUMA : public CHeapObj<mtGC> {
+  // Mapping of available numa ids to some 0-based index which can be used for
+  // fast resource management. I.e. for every numa id provides a unique value in
+  // the range from [0, {# of numa nodes-1}].
+  // For invalid numa id, return G1MemoryNodeManager::InvalidNodeIndex.
+  // So the caller need exception handling.
+  uint* _numa_id_to_index_map;
+  // Length of numa_id to index map.
+  int _len_numa_id_to_index_map;
+
+  // Used to create next numa index which cycles through a range of [0, {# of numa nodes-1}).
+  // This next numa index is used when the given numa index is invalid.
+  volatile uint _next_numa_index;
+
+  // Necessary when touching memory.
+  size_t _page_size;
+
+  // Current active numa ids.
+  int* _numa_ids;
+  // Total number of numa ids.
+  uint _num_active_numa_ids;
+
+  static G1NUMA* _inst;
+
+  // Creates numa id and numa index mapping table of _numa_id_to_index_map.
+  void init_numa_id_to_index_map(const int* numa_ids, uint num_numa_ids);
+
+  // Touch the given memory with specific numa index.
+  void touch_memory_whole(address aligned_address, size_t size_in_bytes, uint numa_index);
+
+  // Touch the given memory with round-robin manner.
+  void touch_memory_roundrobin(address aligned_address, size_t size_in_bytes);
+
+public:
+  G1NUMA() : _numa_id_to_index_map(NULL), _len_numa_id_to_index_map(0), _next_numa_index(0), 
+             _page_size(0), _numa_ids(NULL), _num_active_numa_ids(0) { }
+  ~G1NUMA();
+
+  static G1NUMA* numa() { return _inst; }
+
+  static void set_numa(G1NUMA* numa) {
+    guarantee(_inst == NULL, "Should be called once.");
+
+    _inst = numa;
+  }
+
+  void touch_memory(address aligned_address, size_t size_in_bytes, uint numa_index);
+
+  bool initialize();
+
+  inline bool is_valid_numa_id(int numa_id);
+
+  inline bool is_valid_numa_index(uint numa_index) const;
+
+  inline uint num_active_numa_ids() const;
+
+  // Gets a next valid numa index.
+  inline uint next_numa_index();
+
+  // Gets a next valid numa id.
+  inline int next_numa_id();
+
+  // Returns numa index of the given numa id.
+  // Returns G1MemoryNodeManager::InvalidNodeIndex if the given numa id is invalid.
+  inline uint index_of_numa_id(int numa_id) const;
+
+  // Returns numa id of the given numa index.
+  inline int numa_id_of_index(uint numa_index) const;
+
+  void set_page_size(size_t page_size);
+
+  // Returns current active numa ids.
+  const int* numa_ids() const { return _numa_ids; }
+
+  // Returns numa index of the given address via system call.
+  uint index_of_address(HeapWord* addr) const;
+};
+
+#endif // SHARE_VM_GC_G1_G1NUMA_HPP
--- /dev/null	2019-09-04 10:58:08.784555846 -0700
+++ new/src/hotspot/share/gc/g1/g1NUMA.inline.hpp	2019-09-21 06:25:36.713961190 -0700
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_G1_NUMA_INLINE_HPP
+#define SHARE_VM_GC_G1_NUMA_INLINE_HPP
+
+#include "gc/g1/g1MemoryNodeManager.hpp"
+#include "gc/g1/g1NUMA.hpp"
+#include "runtime/atomic.hpp"
+
+inline bool G1NUMA::is_valid_numa_id(int numa_id) {
+  // Valid numa id should be one of active numa ids.
+  for (uint i = 0; i < _num_active_numa_ids; i++) {
+    if (_numa_ids[i] == numa_id) {
+      return true;
+    }
+  }
+  return false;
+}
+
+inline bool G1NUMA::is_valid_numa_index(uint numa_index) const {
+  // Valid numa index should be less than the number of active numa ids.
+  return numa_index < _num_active_numa_ids;
+}
+
+inline uint G1NUMA::num_active_numa_ids() const {
+  assert(_num_active_numa_ids > 0, "just checking");
+  return _num_active_numa_ids;
+}
+
+inline uint G1NUMA::next_numa_index() {
+  // Overflow of uint does not matter here to get numa index.
+  uint numa_index = Atomic::add(1u, &_next_numa_index);
+  return numa_index % _num_active_numa_ids;
+}
+
+inline int G1NUMA::next_numa_id() {
+  return _numa_ids[next_numa_index()];
+}
+
+inline uint G1NUMA::index_of_numa_id(int numa_id) const {
+  // Don't need call is_valid_numa_id, as _numa_id_to_index_map 
+  // may return G1MemoryNodeManager::InvalidNodeIndex.
+  if (numa_id >= 0 && numa_id < _len_numa_id_to_index_map) {
+    return _numa_id_to_index_map[numa_id];
+  }
+  return G1MemoryNodeManager::InvalidNodeIndex;
+}
+
+inline int G1NUMA::numa_id_of_index(uint numa_index) const {
+  if (is_valid_numa_index(numa_index)) {
+    return _numa_ids[numa_index];
+  }
+  return os::InvalidId;
+}
+
+#endif // SHARE_VM_GC_G1_NUMA_INLINE_HPP
--- /dev/null	2019-09-04 10:58:08.784555846 -0700
+++ new/test/hotspot/jtreg/gc/g1/numa/TestG1NUMATouchRegions.java	2019-09-21 06:25:37.837961151 -0700
@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.g1;
+
+/**
+ * @test TestG1NUMATouchRegions
+ * @summary Ensure the bottom of the given heap regions are properly touched with requested NUMA id.
+ * @key gc
+ * @requires vm.gc.G1
+ * @requires os.family == "linux"
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ *          java.management
+ * @build sun.hotspot.WhiteBox
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ * @run main/othervm -XX:+UseG1GC -Xbootclasspath/a:. -XX:+UseNUMA -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI gc.g1.TestG1NUMATouchRegions
+ */
+
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+import sun.hotspot.WhiteBox;
+
+public class TestG1NUMATouchRegions {
+    enum NUMASupportStatus {
+        NOT_CHECKED,
+        SUPPORT,
+        NOT_SUPPORT
+    };
+
+    static int G1HeapRegionSize1MB = 1;
+    static int G1HeapRegionSize8MB = 8;
+
+    static NUMASupportStatus status = NUMASupportStatus.NOT_CHECKED;
+
+    public static void main(String[] args) throws Exception {
+        // 1. Page size < G1HeapRegionSize
+        //    Test default page with 1MB heap region size
+        testMemoryTouch("-XX:-UseLargePages", G1HeapRegionSize1MB);
+        // 2. Page size > G1HeapRegionSize
+        //    Test large page with 1MB heap region size.
+        testMemoryTouch("-XX:+UseLargePages", G1HeapRegionSize1MB);
+        // 3. Page size < G1HeapRegionSize
+        //    Test large page with 8MB heap region size.
+        testMemoryTouch("-XX:+UseLargePages", G1HeapRegionSize8MB);
+    }
+
+    // On Linux, always UseNUMA is enabled if there is multiple active numa nodes.
+    static NUMASupportStatus checkNUMAIsEnabled(OutputAnalyzer output) {
+        boolean supportNUMA = Boolean.parseBoolean(output.firstMatch("\\bUseNUMA\\b.*?=.*?([a-z]+)", 1));
+        System.out.println("supportNUMA=" + supportNUMA);
+        return supportNUMA ? NUMASupportStatus.SUPPORT : NUMASupportStatus.NOT_SUPPORT;
+    }
+
+    static long parseSizeString(String size) {
+        long multiplier = 1;
+
+        if (size.endsWith("B")) {
+            multiplier = 1;
+        } else if (size.endsWith("K")) {
+            multiplier = 1024;
+        } else if (size.endsWith("M")) {
+            multiplier = 1024 * 1024;
+        } else if (size.endsWith("G")) {
+            multiplier = 1024 * 1024 * 1024;
+        } else {
+            throw new IllegalArgumentException("Expected memory string '" + size + "'to end with either of: B, K, M, G");
+        }
+
+        long longSize = Long.parseUnsignedLong(size.substring(0, size.length() - 1));
+
+        return longSize * multiplier;
+    }
+
+    static long heapPageSize(OutputAnalyzer output) {
+        String HeapPageSizePattern = "Heap:  .*page_size=([^ ]+)";
+        String str = output.firstMatch(HeapPageSizePattern, 1);
+
+        if (str == null) {
+            output.reportDiagnosticSummary();
+            throw new RuntimeException("Match from '" + HeapPageSizePattern + "' got 'null'");
+        }
+
+        return parseSizeString(str);
+    }
+
+    // 1. -UseLargePages: default page, page size < G1HeapRegionSize
+    //    +UseLargePages: large page size <= G1HeapRegionSize
+    //
+    //    Each 'int' represents a numa id of single HeapRegion (bottom page).
+    //    e.g. heap region # : numa id of page
+    //         0 : 00
+    //         1 : 01
+    static void checkCase1Pattern(OutputAnalyzer output, int index, long g1HeapRegionSize, long actualPageSize, int[] memoryNodeIds) throws Exception {
+        StringBuilder sb = new StringBuilder();
+
+        // Append index which means heap region index.
+        sb.append(index);
+        sb.append(" ");
+
+        sb.append("(");
+        sb.append(index);
+        sb.append("): ");
+
+        // Append page node id.
+        sb.append(String.format("%02x", memoryNodeIds[index]));
+
+        output.shouldContain(sb.toString());
+    }
+
+    // 3. +UseLargePages: large page size > G1HeapRegionSize
+    //
+    //    As a OS page is consist of multiple heap regions, log also should be
+    //    printed multiple times for same numa id.
+    //    e.g. 1MB heap region, 2MB page size
+    //         heap region # (page #): numa id of page
+    //         0 (0): 00
+    //         1 (0): 00
+    //         2 (1): 01
+    //         3 (1): 01
+    static void checkCase2Pattern(OutputAnalyzer output, int index, long g1HeapRegionSize, long actualPageSize, int[] memoryNodeIds) throws Exception {
+        StringBuilder sb = new StringBuilder();
+
+        // Append page range.
+        int lines_to_print = (int)(actualPageSize / g1HeapRegionSize);
+        for (int i = 0; i < lines_to_print; i++) {
+            // Append index which means heap region index.
+            sb.append(index * lines_to_print + i);
+            sb.append(" ");
+
+            sb.append("(");
+            sb.append(index);
+            sb.append("): ");
+
+            // Append page node id.
+            sb.append(String.format("%02x", memoryNodeIds[index]));
+
+            output.shouldContain(sb.toString());
+            sb.setLength(0);
+        }
+
+        output.shouldContain(sb.toString());
+    }
+
+    static void checkNUMALog(OutputAnalyzer output, int regionSizeInMB) throws Exception {
+        WhiteBox wb = WhiteBox.getWhiteBox();
+        long g1HeapRegionSize = regionSizeInMB * 1024 * 1024;
+        long actualPageSize = heapPageSize(output);
+        long defaultPageSize = (long)wb.getVMPageSize();
+        int memoryNodeCount = wb.g1ActiveMemoryNodeCount();
+        int[] memoryNodeIds = wb.g1MemoryNodeIds();
+
+        // Check for the first one set of active numa nodes.
+        for (int index = 0; index < memoryNodeCount; index++) {
+            if (actualPageSize <= defaultPageSize) {
+                checkCase1Pattern(output, index, g1HeapRegionSize, actualPageSize, memoryNodeIds);
+            } else {
+                checkCase2Pattern(output, index, g1HeapRegionSize, actualPageSize, memoryNodeIds);
+            }
+        }
+    }
+
+    static void testMemoryTouch(String largePagesSetting, int regionSizeInMB) throws Exception {
+        // Skip testing with message.
+        if (status == NUMASupportStatus.NOT_SUPPORT) {
+            System.out.println("NUMA is not supported");
+            return;
+        }
+
+        ProcessBuilder pb_enabled = ProcessTools.createJavaProcessBuilder(
+                                              "-Xbootclasspath/a:.",
+                                              "-Xlog:gc+heap+numa=debug,pagesize",
+                                              "-XX:+UseG1GC",
+                                              "-Xmx128m",
+                                              "-Xms128m",
+                                              "-XX:+UnlockDiagnosticVMOptions",
+                                              "-XX:+WhiteBoxAPI",
+                                              "-XX:+PrintNUMAIdOfHeapRegions",
+                                              "-XX:+PrintFlagsFinal",
+                                              "-XX:+UseNUMA",
+                                              "-XX:+AlwaysPreTouch",
+                                              largePagesSetting,
+                                              "-XX:G1HeapRegionSize=" + regionSizeInMB + "m",
+                                              "--version");
+        OutputAnalyzer output = new OutputAnalyzer(pb_enabled.start());
+
+        // Check NUMA availability.
+        if (status == NUMASupportStatus.NOT_CHECKED) {
+            status = checkNUMAIsEnabled(output);
+        }
+
+        if (status == NUMASupportStatus.SUPPORT) {
+            checkNUMALog(output, regionSizeInMB);
+        } else {
+            // Exit with message for the first test.
+            System.out.println("NUMA is not supported");
+        }
+    }
+}